diff --git a/RELEASES.json b/RELEASES.json index 0bf3161aaee619bc1a1c0ba295dd28a065b55a15..89e5222ba8bf040b5c1a9533019978f8682f36e9 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,5 +1,5 @@ { - "current": "v53 (89ed2cb)", + "current": "a8583e6", "releases": [ { "version": "v53", diff --git a/VERSION b/VERSION index 10159eae2c3614e78f90d80ad63e1c3a6bbde1ff..1bec4cb0b27a42da559c622b70ade78a21d62496 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v53 (89ed2cb) +a8583e6 diff --git a/api/ai_review.py b/api/ai_review.py index 093c323357f15f25b5301e2db540196f9d7855ee..03f68bd48c54f4543883f221926d84a4cd135c1f 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -1,591 +1,591 @@ -"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). - -⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made -that somebody optionally a MODEL: the engine hands over the record's own values, the review's -prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of -those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit -log a human click writes to, tagged `by: "ai"` with the provider and model that made it. - -⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a -network failure, a slow answer, a malformed answer, or an answer naming a stage the review does -not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would -have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing -the work. Nothing here can move a card somewhere the review does not already permit. - -⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is -`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is -last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` -pins one; `AIOS_AI_MODEL` overrides the model. - -⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH -USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting -the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. -The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the -integrator can overturn it deliberately rather than by drift"*. This is that overturn. - -Its two reasons were real and are both answered rather than waved away: - 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is - not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers - the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no - Anthropic knowledge — it lost some. - 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the - import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the - same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH - manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. -⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that -half of the original reasoning never expired. - -The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and -`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an -empty content list, so code that indexes `content[0]` unconditionally breaks on it. -""" -from __future__ import annotations - -import json -import os -import re - -import requests - -#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. -PROVIDERS = [ - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED - # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent - # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo - # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. - {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", - "url": "https://api.groq.com/openai/v1/chat/completions", - "model": "openai/gpt-oss-120b"}, - {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", - "url": "https://api.cerebras.ai/v1/chat/completions", - "model": "gpt-oss-120b"}, - {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", - "url": "https://openrouter.ai/api/v1/chat/completions", - "model": "openai/gpt-4o-mini"}, - # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as - # the backstop for a one-line classification, and this is the cheapest current Claude that - # does it well. A bigger model here would be spending the owner's money to pick between two - # labels it already has in front of it. - {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", - "url": "https://api.anthropic.com/v1/messages", - # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, - # and a deployment whose side rungs are out of credit can raise the tier without a release. - "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, -] -ANTHROPIC_VERSION = "2023-06-01" -TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) -MAX_FIELD_CHARS = 200 # per value handed to the model -MAX_FIELDS = 30 # columns handed to the model -MAX_REASON = 200 - - -def ladder(): - """The providers that are actually usable here, in order. Empty = the feature is off.""" - pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() - live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] - if pin: - live = [p for p in live if p["name"] == pin] - return live - - -def configured(): - return bool(ladder()) - - -def _record_text(row, fields): - """The record, as the model sees it. Values are truncated and the column set is bounded — - an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not - need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is - sitting at would be the model reading its own question back.""" - keys = [k for k in (fields or list((row or {}).keys())) - if not str(k).startswith("stage_")][:MAX_FIELDS] - lines = [] - for k in keys: - v = (row or {}).get(k) - if v is None or str(v).strip() == "": - continue - lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") - return "\n".join(lines) or "(this record has no filled-in values)" - - -def _instruction(prompt, options, label): - return ( - f"You are deciding what happens to one record waiting at a review step called " - f"{label!r} in a workflow.\n\n" - f"The person who built this workflow told you: {prompt}\n\n" - f"Choose EXACTLY ONE of these next steps, by its exact name:\n" - + "\n".join(f"- {o}" for o in options) - + "\n\nAnswer with one line of JSON and nothing else:\n" - '{"choice": "", "reason": ""}\n' - "If the record does not give you enough to decide, answer " - '{"choice": "", "reason": "why not"} and a person will decide instead.' - ) - - -def _parse(text, options): - """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is - DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss - resolved by string distance is how a card ends up somewhere nobody authorised.""" - raw = str(text or "").strip() - obj = None - m = re.search(r"\{.*\}", raw, re.S) - if m: - try: - obj = json.loads(m.group(0)) - except (ValueError, TypeError): - obj = None - if not isinstance(obj, dict): - return "", "" - choice = str(obj.get("choice") or "").strip() - reason = str(obj.get("reason") or "").strip()[:MAX_REASON] - for opt in options: - if choice.lower() == str(opt).lower(): - return str(opt), reason # the OFFERED spelling wins, never the model's - return "", reason - - -# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. -# -# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s -# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token -# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach -# this transport rather than be bolted onto its caller — the caller never sees the body. -# -# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The -# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), -# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers -# shape this repo keeps paying for. -def _call_openai(p, model, system, user, timeout): - r = requests.post(p["url"], timeout=timeout, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": model, "max_tokens": 300, "temperature": 0, - "messages": [{"role": "system", "content": system}, - {"role": "user", "content": user}]}) - if r.status_code >= 400: - # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused - # before running the model has nothing to bill. The call is still COUNTED by `decide`. - return "", f"{p['name']} answered {r.status_code}", None - body = r.json() - choices = body.get("choices") or [] - if not choices: - return "", f"{p['name']} returned no choices", body - return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body - - -#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. -#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say -#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. -LAST_ANTHROPIC_TRANSPORT = "" - - -def _call_anthropic(p, model, system, user, timeout): - """One classification turn on the Messages API, through `providers.anthropic_send`. - - ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which - prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is - absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises - both transports to `(status, body)`, so every line below this call is untouched. - """ - global LAST_ANTHROPIC_TRANSPORT - import providers as _prov # noqa: PLC0415 - # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is - # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. - req = _prov.anthropic_request( - model=model, key=os.environ[p["env"]].strip(), system=system, - messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) - status, body, transport = _prov.anthropic_send(req, timeout=timeout) - LAST_ANTHROPIC_TRANSPORT = transport - if status >= 400 or status == 0: - # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the - # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" - # (which is what this line used to say, and what the owner quoted back at us) cannot recur. - return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None - body = body or {} - # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so - # reading content[0] before this check turns a refusal into an IndexError inside a run. - # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. - if body.get("stop_reason") == "refusal": - return "", "anthropic declined to answer this record", body - parts = [b.get("text") or "" for b in (body.get("content") or []) - if isinstance(b, dict) and b.get("type") == "text"] - if not parts: - return "", "anthropic returned no text", body - return "".join(parts), "", body - - -def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): - """Pick this record's next stage. Returns `(choice, meta)`. - - `choice` is "" whenever a person should decide — which is every failure mode there is. - `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. - - ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because - this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it - HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as - unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for - why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI - dependency). - """ - import usage_ledger # noqa: PLC0415 - opts = [str(o) for o in (options or []) if str(o).strip()] - if not opts: - return "", {"problem": "the review offers no next stages"} - if not str(prompt or "").strip(): - return "", {"problem": "the review has no prompt for the model to follow"} - live = ladder() - if not live: - return "", {"problem": "no AI provider is configured on this deployment"} - system = _instruction(prompt, opts, label) - user = "Here is the record:\n\n" + _record_text(row, fields) - tmo = float(timeout or TIMEOUT_SECONDS) - override = (os.environ.get("AIOS_AI_MODEL") or "").strip() - problems = [] - for p in live: - model = override or p["model"] - body = None - try: - text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( - p, model, system, user, tmo) - except Exception as e: # noqa: BLE001 - text, err = "", f"{p['name']} failed: {type(e).__name__}" - # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. - # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an - # unusable answer. A meter that books only successes reports a cheap week for a run that - # spent its budget being declined — and a declined call is billed. The only path that does - # NOT record is a transport that never reached the vendor (`body is None`), which spent - # nothing. - if body is not None: - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record("ai_review", p["name"], model, ins, outs, - total=usage_ledger.total_from(body), st=st, user=user) - if err: - problems.append(err) - continue # ladder: a dead provider degrades to the next one - choice, reason = _parse(text, opts) - if not choice: - # The provider ANSWERED and declined (or answered unusably). That is a decision about - # this record, not a fault in the provider, so it does NOT fall through to a more - # expensive one — the card goes to a human, which is what the model just asked for. - return "", {"provider": p["name"], "model": model, - "problem": reason or "the model did not choose one of the stages"} - return choice, {"provider": p["name"], "model": model, "reason": reason} - return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} - - -# ══════════════════════════════════════════════════════════════════ the FLOW WRITER -# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. -# -# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the -# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: -# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the -# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object -# in that vocabulary. -# -# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This -# is the one module in the API package that already holds an LLM ladder and its credentials, so a -# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs -# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 -# shipped three finished routers 404-dead behind green gates. The DOOR is a route on -# `routes_automation.py`, which is already mounted. -# -# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds -# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are -# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema -# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters -# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise -# and answers 200, so a draft accepted without the diff would show a person a flow that is not the -# flow the model described, with nothing anywhere saying so. -# -# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door -# can be proven end to end with no API key and no spend. A gate that can only run where a -# credential exists is a gate that never runs. - -#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: -#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you -#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, -#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. -FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") -MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 -MAX_PROMPT_CHARS = 2000 - - -def flow_providers(pin=None): - """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. - - ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and - the draft door used to read `prompt` off the body and nothing else — so the key the client sent - was accepted and dropped, and the picker was a control over nothing. - ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the - ladder stopped offering must not turn every later draft into an error; the caller is told which - rung actually answered, which is the honest half. - """ - by_name = {p["name"]: p for p in PROVIDERS} - live = [n for n in FLOW_PROVIDER_ORDER - if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] - wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() - if wanted and wanted in live: - return [by_name[wanted]] - return [by_name[n] for n in live] - - -def flow_schema(kinds, trigger_keys, table_keys): - """The tool schema the model fills — the catalog's OWN key lists as enums. - - ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the - transport rather than by a check afterwards. A model asked for "any action name" invents - `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a - 400 from `clean_actions` carrying a sentence about a word the person never typed. - ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a - schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a - transport failure (`routes_query._spec_schema` carries the same note, for the same measured - reason). - """ - return { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": ["flow", "refused"], - "description": "refused = this cannot be built from the steps available"}, - "refusal": {"type": "string", - "description": "when kind=refused: ONE plain sentence naming what is " - "missing, in the words a non-technical person would use"}, - "name": {"type": "string", "description": "a short title for the automation"}, - "trigger": {"type": "string", "enum": sorted(trigger_keys), - "description": "what starts this automation"}, - "table": {"type": "string", "enum": sorted(table_keys), - "description": "the database whose records this flow walks, if any"}, - "actions": { - "type": "array", - "maxItems": MAX_DRAFT_ACTIONS, - "items": { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": sorted(kinds)}, - "why": {"type": "string", - "description": "one short sentence: why this step is here"}, - "config": {"type": "object", "additionalProperties": True, - "description": "the step's settings, using ONLY the keys named " - "for that kind in the system message"}, - }, - "required": ["kind"], - }, - }, - }, - "required": ["kind"], - } - - -def flow_system_prompt(catalog, required, triggers, tables): - """What the model is told it may build with — DERIVED, never written down twice. - - Every list here is the server's own: the catalog rows the menu paints, the required-key table - the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases - with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to - the catalog is offered here on the same deploy and a kind removed stops being offered. - """ - lines = ["You build small automations for a business tool. You are given the EXACT set of " - "steps this tool can perform. You may use nothing else.", - "", "THE STEPS YOU MAY USE:"] - # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the - # PHRASE is the only human wording of that requirement anywhere — the one the runner's own - # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a - # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing - # what to call the setting. Handing it one and hoping is how a draft comes back configured - # against a key the validator drops. - req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} - for row in catalog or []: - if not row.get("ready"): - continue - kind = str(row.get("kind") or "") - need = req.get(kind) or [] - lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" - + (f" REQUIRED settings: {', '.join(need)}" if need else "")) - lines += ["", "WHAT CAN START AN AUTOMATION:"] - for t in triggers or []: - if t.get("planned") or not t.get("ready", True): - continue - lines.append(f"- {t.get('key')}: {t.get('label')}") - lines += ["", "THE DATABASES THIS PERSON HAS:"] - for t in (tables or [])[:40]: - cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) - lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") - if not tables: - lines.append("- (none - do not name a database)") - lines += [ - "", - "RULES:", - "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " - "answer kind=refused and say plainly which capability is missing.", - "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " - "rather than inventing a web address, a CSS selector or a column name.", - "3. A web address must start with http:// or https://.", - "4. To use a value from the record the flow is walking, write it as {{Column name}}.", - "5. Name a database only from the list above, by its key.", - "6. Keep it short. Fewer steps that work beat more steps that guess.", - ] - return "\n".join(lines) - - -def _flow_from_tool_call(obj): - """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" - if not isinstance(obj, dict): - return None, "the assistant's answer could not be read" - if str(obj.get("kind") or "") == "refused": - return None, (str(obj.get("refusal") or "").strip() - or "the assistant could not build this from the steps available") - acts = obj.get("actions") - if not isinstance(acts, list) or not acts: - return None, ("the assistant did not produce any steps - try describing what should " - "happen, one action at a time") - out = [] - for a in acts[:MAX_DRAFT_ACTIONS]: - if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): - continue - cfg = a.get("config") - out.append({"kind": str(a["kind"]).strip(), - "config": cfg if isinstance(cfg, dict) else {}, - "why": str(a.get("why") or "").strip()[:200]}) - if not out: - return None, "the assistant's steps could not be read" - return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", - "trigger": str(obj.get("trigger") or "").strip(), - "table": str(obj.get("table") or "").strip(), - # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, - # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need - # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the - # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past - # the twelfth and said nothing, so a model that answered with a twenty-step journey had - # eight steps deleted between the answer and the screen, with no key anywhere in the - # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this - # ceiling is the draft door's own, which makes reporting it the whole obligation. - # ⚠ Found by the verifier that checked this ticket, not by a gate. - "asked": len([a for a in acts if isinstance(a, dict)]), - "actions": out}, "" - - -def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, - st=None, user="", model=None): - """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. - - Exactly one of `draft` and `refusal_sentence` is truthy — the same contract - `web_agent.run_step` keeps, so a caller has no third case to get wrong. - - ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for - the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and - `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and - D asserts it; the two keywords are what D has to pass for the line to be attributable. - """ - # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. - # It parses either way; a reader scanning for the declaration does not look inside a `for`. - global LAST_ANTHROPIC_TRANSPORT - import usage_ledger # noqa: PLC0415 - text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] - if not text: - return None, "type what you want the automation to do", None - kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) - if not kinds: - return None, "this deployment offers no automation steps to build with", None - trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) - if t.get("key") and not t.get("planned")}) or ["manual"] - table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] - tools = [{"type": "function", "function": { - "name": "build_automation", - "description": "Emit the automation, or refuse.", - "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] - messages = [{"role": "system", - "content": flow_system_prompt(catalog, required, triggers, tables)}, - {"role": "user", "content": text}] - - if chat is not None: - # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal - # branches as a live call; only the transport is replaced. - draft, refusal = _flow_from_tool_call(chat(messages, tools)) - return draft, refusal, "injected" - - provs = flow_providers(model) - if not provs: - # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was - # never built [[flag-shipped-without-its-writer]]. - return None, ("the assistant is not configured on this deployment, so an automation cannot " - "be drafted from a description yet"), None - tmo = float(timeout or TIMEOUT_SECONDS) - problems = [] - import providers as _prov - for p in provs: - # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read - # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, - # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the - # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* - # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: - # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter - # 402), so without this branch the drafter cannot answer at all. - # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this - # ladder runs. The parameter is the caller's to send, which is why the wire takes it. - # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch - # below reads one thing. The anthropic side gets there through `anthropic_send` (official - # SDK when installed, the identical raw POST when not); the openai side still posts here - # because there is no shared SDK across three different vendors on that wire. - global LAST_ANTHROPIC_TRANSPORT - try: - if p["shape"] == "anthropic": - req = _prov.anthropic_request( - model=p["model"], key=os.environ[p["env"]].strip(), system=None, - messages=messages, tools=tools, max_tokens=1500, tool_choice="required") - status, body, transport = _prov.anthropic_send(req, timeout=tmo) - LAST_ANTHROPIC_TRANSPORT = transport - else: - r = requests.post(p["url"], timeout=tmo, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1500}) - status = r.status_code - body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: {type(e).__name__}") - continue - if status != 200: - # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off - # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and - # the memo makes the next turn SKIP the empty account instead of paying for it again. - # ⚠ The body is re-serialised for the substring test because the SDK path never had a - # `.text` — that is the one thing this normalisation costs, and it costs nothing else. - detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] - if _prov.is_credit_failure(status, detail): - _prov.mark_no_credit(p["name"]) - problems.append(_prov.refusal_sentence(p["name"], status, detail)) - continue - try: - body = body or {} - if p["shape"] == "anthropic": - _text, args, _refused = _prov.anthropic_read(body) - if _refused: - problems.append(f"{p['name']}: {_refused}") - continue - else: - calls = (((body.get("choices") or [{}])[0].get("message") or {}) - .get("tool_calls") or []) - args = json.loads(calls[0]["function"]["arguments"]) if calls else None - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") - continue - # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE - # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer - # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot - # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that - # branch by construction. The 200-with-junk case is rare and named here rather than - # silently rounded to zero. - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, - total=usage_ledger.total_from(body), st=st, user=user) - draft, refusal = _flow_from_tool_call(args) - if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": - # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more - # expensive rung — the model has just said this cannot be built. `decide()` takes the - # same posture for the same reason. - return None, refusal, p["name"] - if draft is not None: - return draft, "", p["name"] - problems.append(f"{p['name']}: {refusal}") - return None, ("; ".join(problems)[:300] or "no provider answered"), None +"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). + +⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made +that somebody optionally a MODEL: the engine hands over the record's own values, the review's +prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of +those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit +log a human click writes to, tagged `by: "ai"` with the provider and model that made it. + +⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a +network failure, a slow answer, a malformed answer, or an answer naming a stage the review does +not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would +have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing +the work. Nothing here can move a card somewhere the review does not already permit. + +⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is +`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is +last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` +pins one; `AIOS_AI_MODEL` overrides the model. + +⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH +USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting +the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. +The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the +integrator can overturn it deliberately rather than by drift"*. This is that overturn. + +Its two reasons were real and are both answered rather than waved away: + 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is + not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers + the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no + Anthropic knowledge — it lost some. + 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the + import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the + same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH + manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. +⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that +half of the original reasoning never expired. + +The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and +`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an +empty content list, so code that indexes `content[0]` unconditionally breaks on it. +""" +from __future__ import annotations + +import json +import os +import re + +import requests + +#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. +PROVIDERS = [ + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED + # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent + # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo + # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. + {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", + "url": "https://api.groq.com/openai/v1/chat/completions", + "model": "openai/gpt-oss-120b"}, + {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", + "url": "https://api.cerebras.ai/v1/chat/completions", + "model": "gpt-oss-120b"}, + {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", + "url": "https://openrouter.ai/api/v1/chat/completions", + "model": "openai/gpt-4o-mini"}, + # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as + # the backstop for a one-line classification, and this is the cheapest current Claude that + # does it well. A bigger model here would be spending the owner's money to pick between two + # labels it already has in front of it. + {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", + "url": "https://api.anthropic.com/v1/messages", + # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, + # and a deployment whose side rungs are out of credit can raise the tier without a release. + "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, +] +ANTHROPIC_VERSION = "2023-06-01" +TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) +MAX_FIELD_CHARS = 200 # per value handed to the model +MAX_FIELDS = 30 # columns handed to the model +MAX_REASON = 200 + + +def ladder(): + """The providers that are actually usable here, in order. Empty = the feature is off.""" + pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() + live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] + if pin: + live = [p for p in live if p["name"] == pin] + return live + + +def configured(): + return bool(ladder()) + + +def _record_text(row, fields): + """The record, as the model sees it. Values are truncated and the column set is bounded — + an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not + need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is + sitting at would be the model reading its own question back.""" + keys = [k for k in (fields or list((row or {}).keys())) + if not str(k).startswith("stage_")][:MAX_FIELDS] + lines = [] + for k in keys: + v = (row or {}).get(k) + if v is None or str(v).strip() == "": + continue + lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") + return "\n".join(lines) or "(this record has no filled-in values)" + + +def _instruction(prompt, options, label): + return ( + f"You are deciding what happens to one record waiting at a review step called " + f"{label!r} in a workflow.\n\n" + f"The person who built this workflow told you: {prompt}\n\n" + f"Choose EXACTLY ONE of these next steps, by its exact name:\n" + + "\n".join(f"- {o}" for o in options) + + "\n\nAnswer with one line of JSON and nothing else:\n" + '{"choice": "", "reason": ""}\n' + "If the record does not give you enough to decide, answer " + '{"choice": "", "reason": "why not"} and a person will decide instead.' + ) + + +def _parse(text, options): + """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is + DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss + resolved by string distance is how a card ends up somewhere nobody authorised.""" + raw = str(text or "").strip() + obj = None + m = re.search(r"\{.*\}", raw, re.S) + if m: + try: + obj = json.loads(m.group(0)) + except (ValueError, TypeError): + obj = None + if not isinstance(obj, dict): + return "", "" + choice = str(obj.get("choice") or "").strip() + reason = str(obj.get("reason") or "").strip()[:MAX_REASON] + for opt in options: + if choice.lower() == str(opt).lower(): + return str(opt), reason # the OFFERED spelling wins, never the model's + return "", reason + + +# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. +# +# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s +# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token +# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach +# this transport rather than be bolted onto its caller — the caller never sees the body. +# +# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The +# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), +# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers +# shape this repo keeps paying for. +def _call_openai(p, model, system, user, timeout): + r = requests.post(p["url"], timeout=timeout, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": model, "max_tokens": 300, "temperature": 0, + "messages": [{"role": "system", "content": system}, + {"role": "user", "content": user}]}) + if r.status_code >= 400: + # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused + # before running the model has nothing to bill. The call is still COUNTED by `decide`. + return "", f"{p['name']} answered {r.status_code}", None + body = r.json() + choices = body.get("choices") or [] + if not choices: + return "", f"{p['name']} returned no choices", body + return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body + + +#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. +#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say +#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. +LAST_ANTHROPIC_TRANSPORT = "" + + +def _call_anthropic(p, model, system, user, timeout): + """One classification turn on the Messages API, through `providers.anthropic_send`. + + ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which + prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is + absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises + both transports to `(status, body)`, so every line below this call is untouched. + """ + global LAST_ANTHROPIC_TRANSPORT + import providers as _prov # noqa: PLC0415 + # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is + # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. + req = _prov.anthropic_request( + model=model, key=os.environ[p["env"]].strip(), system=system, + messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) + status, body, transport = _prov.anthropic_send(req, timeout=timeout) + LAST_ANTHROPIC_TRANSPORT = transport + if status >= 400 or status == 0: + # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the + # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" + # (which is what this line used to say, and what the owner quoted back at us) cannot recur. + return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None + body = body or {} + # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so + # reading content[0] before this check turns a refusal into an IndexError inside a run. + # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. + if body.get("stop_reason") == "refusal": + return "", "anthropic declined to answer this record", body + parts = [b.get("text") or "" for b in (body.get("content") or []) + if isinstance(b, dict) and b.get("type") == "text"] + if not parts: + return "", "anthropic returned no text", body + return "".join(parts), "", body + + +def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): + """Pick this record's next stage. Returns `(choice, meta)`. + + `choice` is "" whenever a person should decide — which is every failure mode there is. + `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. + + ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because + this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it + HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as + unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for + why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI + dependency). + """ + import usage_ledger # noqa: PLC0415 + opts = [str(o) for o in (options or []) if str(o).strip()] + if not opts: + return "", {"problem": "the review offers no next stages"} + if not str(prompt or "").strip(): + return "", {"problem": "the review has no prompt for the model to follow"} + live = ladder() + if not live: + return "", {"problem": "no AI provider is configured on this deployment"} + system = _instruction(prompt, opts, label) + user = "Here is the record:\n\n" + _record_text(row, fields) + tmo = float(timeout or TIMEOUT_SECONDS) + override = (os.environ.get("AIOS_AI_MODEL") or "").strip() + problems = [] + for p in live: + model = override or p["model"] + body = None + try: + text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( + p, model, system, user, tmo) + except Exception as e: # noqa: BLE001 + text, err = "", f"{p['name']} failed: {type(e).__name__}" + # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. + # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an + # unusable answer. A meter that books only successes reports a cheap week for a run that + # spent its budget being declined — and a declined call is billed. The only path that does + # NOT record is a transport that never reached the vendor (`body is None`), which spent + # nothing. + if body is not None: + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("ai_review", p["name"], model, ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + if err: + problems.append(err) + continue # ladder: a dead provider degrades to the next one + choice, reason = _parse(text, opts) + if not choice: + # The provider ANSWERED and declined (or answered unusably). That is a decision about + # this record, not a fault in the provider, so it does NOT fall through to a more + # expensive one — the card goes to a human, which is what the model just asked for. + return "", {"provider": p["name"], "model": model, + "problem": reason or "the model did not choose one of the stages"} + return choice, {"provider": p["name"], "model": model, "reason": reason} + return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} + + +# ══════════════════════════════════════════════════════════════════ the FLOW WRITER +# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. +# +# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the +# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: +# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the +# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object +# in that vocabulary. +# +# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This +# is the one module in the API package that already holds an LLM ladder and its credentials, so a +# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs +# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 +# shipped three finished routers 404-dead behind green gates. The DOOR is a route on +# `routes_automation.py`, which is already mounted. +# +# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds +# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are +# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema +# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters +# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise +# and answers 200, so a draft accepted without the diff would show a person a flow that is not the +# flow the model described, with nothing anywhere saying so. +# +# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door +# can be proven end to end with no API key and no spend. A gate that can only run where a +# credential exists is a gate that never runs. + +#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: +#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you +#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, +#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. +FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") +MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 +MAX_PROMPT_CHARS = 2000 + + +def flow_providers(pin=None): + """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. + + ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and + the draft door used to read `prompt` off the body and nothing else — so the key the client sent + was accepted and dropped, and the picker was a control over nothing. + ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the + ladder stopped offering must not turn every later draft into an error; the caller is told which + rung actually answered, which is the honest half. + """ + by_name = {p["name"]: p for p in PROVIDERS} + live = [n for n in FLOW_PROVIDER_ORDER + if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] + wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() + if wanted and wanted in live: + return [by_name[wanted]] + return [by_name[n] for n in live] + + +def flow_schema(kinds, trigger_keys, table_keys): + """The tool schema the model fills — the catalog's OWN key lists as enums. + + ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the + transport rather than by a check afterwards. A model asked for "any action name" invents + `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a + 400 from `clean_actions` carrying a sentence about a word the person never typed. + ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a + schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a + transport failure (`routes_query._spec_schema` carries the same note, for the same measured + reason). + """ + return { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": ["flow", "refused"], + "description": "refused = this cannot be built from the steps available"}, + "refusal": {"type": "string", + "description": "when kind=refused: ONE plain sentence naming what is " + "missing, in the words a non-technical person would use"}, + "name": {"type": "string", "description": "a short title for the automation"}, + "trigger": {"type": "string", "enum": sorted(trigger_keys), + "description": "what starts this automation"}, + "table": {"type": "string", "enum": sorted(table_keys), + "description": "the database whose records this flow walks, if any"}, + "actions": { + "type": "array", + "maxItems": MAX_DRAFT_ACTIONS, + "items": { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": sorted(kinds)}, + "why": {"type": "string", + "description": "one short sentence: why this step is here"}, + "config": {"type": "object", "additionalProperties": True, + "description": "the step's settings, using ONLY the keys named " + "for that kind in the system message"}, + }, + "required": ["kind"], + }, + }, + }, + "required": ["kind"], + } + + +def flow_system_prompt(catalog, required, triggers, tables): + """What the model is told it may build with — DERIVED, never written down twice. + + Every list here is the server's own: the catalog rows the menu paints, the required-key table + the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases + with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to + the catalog is offered here on the same deploy and a kind removed stops being offered. + """ + lines = ["You build small automations for a business tool. You are given the EXACT set of " + "steps this tool can perform. You may use nothing else.", + "", "THE STEPS YOU MAY USE:"] + # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the + # PHRASE is the only human wording of that requirement anywhere — the one the runner's own + # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a + # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing + # what to call the setting. Handing it one and hoping is how a draft comes back configured + # against a key the validator drops. + req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} + for row in catalog or []: + if not row.get("ready"): + continue + kind = str(row.get("kind") or "") + need = req.get(kind) or [] + lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" + + (f" REQUIRED settings: {', '.join(need)}" if need else "")) + lines += ["", "WHAT CAN START AN AUTOMATION:"] + for t in triggers or []: + if t.get("planned") or not t.get("ready", True): + continue + lines.append(f"- {t.get('key')}: {t.get('label')}") + lines += ["", "THE DATABASES THIS PERSON HAS:"] + for t in (tables or [])[:40]: + cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) + lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") + if not tables: + lines.append("- (none - do not name a database)") + lines += [ + "", + "RULES:", + "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " + "answer kind=refused and say plainly which capability is missing.", + "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " + "rather than inventing a web address, a CSS selector or a column name.", + "3. A web address must start with http:// or https://.", + "4. To use a value from the record the flow is walking, write it as {{Column name}}.", + "5. Name a database only from the list above, by its key.", + "6. Keep it short. Fewer steps that work beat more steps that guess.", + ] + return "\n".join(lines) + + +def _flow_from_tool_call(obj): + """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" + if not isinstance(obj, dict): + return None, "the assistant's answer could not be read" + if str(obj.get("kind") or "") == "refused": + return None, (str(obj.get("refusal") or "").strip() + or "the assistant could not build this from the steps available") + acts = obj.get("actions") + if not isinstance(acts, list) or not acts: + return None, ("the assistant did not produce any steps - try describing what should " + "happen, one action at a time") + out = [] + for a in acts[:MAX_DRAFT_ACTIONS]: + if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): + continue + cfg = a.get("config") + out.append({"kind": str(a["kind"]).strip(), + "config": cfg if isinstance(cfg, dict) else {}, + "why": str(a.get("why") or "").strip()[:200]}) + if not out: + return None, "the assistant's steps could not be read" + return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", + "trigger": str(obj.get("trigger") or "").strip(), + "table": str(obj.get("table") or "").strip(), + # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, + # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need + # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the + # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past + # the twelfth and said nothing, so a model that answered with a twenty-step journey had + # eight steps deleted between the answer and the screen, with no key anywhere in the + # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this + # ceiling is the draft door's own, which makes reporting it the whole obligation. + # ⚠ Found by the verifier that checked this ticket, not by a gate. + "asked": len([a for a in acts if isinstance(a, dict)]), + "actions": out}, "" + + +def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, + st=None, user="", model=None): + """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. + + Exactly one of `draft` and `refusal_sentence` is truthy — the same contract + `web_agent.run_step` keeps, so a caller has no third case to get wrong. + + ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for + the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and + `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and + D asserts it; the two keywords are what D has to pass for the line to be attributable. + """ + # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. + # It parses either way; a reader scanning for the declaration does not look inside a `for`. + global LAST_ANTHROPIC_TRANSPORT + import usage_ledger # noqa: PLC0415 + text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] + if not text: + return None, "type what you want the automation to do", None + kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) + if not kinds: + return None, "this deployment offers no automation steps to build with", None + trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) + if t.get("key") and not t.get("planned")}) or ["manual"] + table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] + tools = [{"type": "function", "function": { + "name": "build_automation", + "description": "Emit the automation, or refuse.", + "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] + messages = [{"role": "system", + "content": flow_system_prompt(catalog, required, triggers, tables)}, + {"role": "user", "content": text}] + + if chat is not None: + # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal + # branches as a live call; only the transport is replaced. + draft, refusal = _flow_from_tool_call(chat(messages, tools)) + return draft, refusal, "injected" + + provs = flow_providers(model) + if not provs: + # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was + # never built [[flag-shipped-without-its-writer]]. + return None, ("the assistant is not configured on this deployment, so an automation cannot " + "be drafted from a description yet"), None + tmo = float(timeout or TIMEOUT_SECONDS) + problems = [] + import providers as _prov + for p in provs: + # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read + # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, + # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the + # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* + # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: + # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter + # 402), so without this branch the drafter cannot answer at all. + # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this + # ladder runs. The parameter is the caller's to send, which is why the wire takes it. + # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch + # below reads one thing. The anthropic side gets there through `anthropic_send` (official + # SDK when installed, the identical raw POST when not); the openai side still posts here + # because there is no shared SDK across three different vendors on that wire. + global LAST_ANTHROPIC_TRANSPORT + try: + if p["shape"] == "anthropic": + req = _prov.anthropic_request( + model=p["model"], key=os.environ[p["env"]].strip(), system=None, + messages=messages, tools=tools, max_tokens=1500, tool_choice="required") + status, body, transport = _prov.anthropic_send(req, timeout=tmo) + LAST_ANTHROPIC_TRANSPORT = transport + else: + r = requests.post(p["url"], timeout=tmo, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1500}) + status = r.status_code + body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: {type(e).__name__}") + continue + if status != 200: + # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off + # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and + # the memo makes the next turn SKIP the empty account instead of paying for it again. + # ⚠ The body is re-serialised for the substring test because the SDK path never had a + # `.text` — that is the one thing this normalisation costs, and it costs nothing else. + detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] + if _prov.is_credit_failure(status, detail): + _prov.mark_no_credit(p["name"]) + problems.append(_prov.refusal_sentence(p["name"], status, detail)) + continue + try: + body = body or {} + if p["shape"] == "anthropic": + _text, args, _refused = _prov.anthropic_read(body) + if _refused: + problems.append(f"{p['name']}: {_refused}") + continue + else: + calls = (((body.get("choices") or [{}])[0].get("message") or {}) + .get("tool_calls") or []) + args = json.loads(calls[0]["function"]["arguments"]) if calls else None + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") + continue + # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE + # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer + # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot + # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that + # branch by construction. The 200-with-junk case is rare and named here rather than + # silently rounded to zero. + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + draft, refusal = _flow_from_tool_call(args) + if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": + # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more + # expensive rung — the model has just said this cannot be built. `decide()` takes the + # same posture for the same reason. + return None, refusal, p["name"] + if draft is not None: + return draft, "", p["name"] + problems.append(f"{p['name']}: {refusal}") + return None, ("; ".join(problems)[:300] or "no provider answered"), None diff --git a/api/automation_engine.py b/api/automation_engine.py index bd520c7ed4da682509e47e91e1712ab64a9880fd..92e522efb35c400abb654023a45bee6d1b25a796 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -1,961 +1,961 @@ -"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. - -THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant -store and a RUN that lives in this process. The definition is durable, small and rarely written; -the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file -is built to avoid, and it has two halves: - - * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the - store outlives the process that wrote it, so a container restart mid-run leaves an automation - that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives - in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written - exactly ONCE per run, at the end. - * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s - floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape - by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes - its whole result first and commits it in a single `update` callback. - -WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not -deployed, so its extraction core is copied into this file (the wave doc's instruction), ported -httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with -it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: -the skill version takes a URL from a developer on a CLI; this takes one from a request body. -""" -from __future__ import annotations - -import copy -import datetime as _dt -import hashlib -import hmac -import ipaddress -import json -import os -import re -import socket -import threading -import time -from urllib.parse import urljoin, urlparse - -import requests - -# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability -# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — -# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the -# connector that routes, which is the shape the split was for: this file no longer knows that -# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore -# a signal, not a convenience — it means something in the automation runtime started making a -# vendor decision, and that belongs one layer down. - -try: # the parser the extraction half needs - from bs4 import BeautifulSoup -except Exception: # pragma: no cover — deploy lag; see mailbox ② - BeautifulSoup = None - -# --------------------------------------------------------------------------------------------- -# THE BUCKET (C4-AUTO) -# --------------------------------------------------------------------------------------------- - -#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. -STORE_KEY = "automations" -#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through -#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for -#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the -#: session-D mailbox as a finding against A's file rather than fixed from here. -UT_STORE_KEY = "user_tables" -UT_PREFIX = "ut_" - -MAX_AUTOMATIONS = 40 -MAX_RUNS = 20 # trimmed history per automation -MAX_NAME = 80 -#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x -#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it -#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently -#: walking 5,000 of 32,826 connected rows. It was. -#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from -#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no -#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is -#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. -#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing -#: should not also be the widest one. And do not delete it — D-143's own row says the editable -#: substrate still needs a bound. -MAX_UT_ROWS = 5000 -MAX_UT_TABLES = 40 - -#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. -#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; -#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real -#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — -#: indistinguishable on every surface from a run that worked, and a strong candidate for why the -#: owner's enrichment kept appearing to do nothing. -#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better -#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. -ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" - -#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. -#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the -#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull -#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's -#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour -#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still -#: report success. A time series that stops after five days without saying so is worse than one -#: that was never built. -#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE -#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: -#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day -#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts -#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had -#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the -#: silent version would have flatlined a chart inside a fortnight with a green dot over it. -#: -#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises -#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, -#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 -#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a -#: smaller number. -#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) -MAX_UT_IG_ROWS = 200_000 -IG_TABLE_PREFIX = "ut_ig_" - -#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was -#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — -#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY -#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would -#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket -#: serialisation cost, purely by an accident of naming. Naming the append tables is the version -#: that stays true when the next `ut_ig_*` table is not one. -IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" -IG_POSTS_TABLE = "ut_ig_posts" -IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" -IG_COMMENTS_TABLE = "ut_ig_comments" - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY -# --------------------------------------------------------------------------------------------- -# R1: FULL PARITY — profile AND posts AND comments, not a profile tier. -# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live -# Instagram rows, and the two schemas may DIVERGE where the vendors do. -# -# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is -# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity -# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`. -# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on -# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video -# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables -# holding hundreds of thousands of rows is a migration on production data to buy a shared grid -# nobody asked for. R2 chose the version with no migration. -# -# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn -# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST -# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says -# `tt_id` rather than inheriting a column labelled "Instagram id". -TT_TABLE_PREFIX = "ut_tt_" -TT_PROFILE_TABLE = "ut_tt_profile" -TT_SNAPSHOTS_TABLE = "ut_tt_snapshots" -TT_POSTS_TABLE = "ut_tt_posts" -TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots" -TT_COMMENTS_TABLE = "ut_tt_comments" - -AUTOMATION_RECORD_MODE = "automation" -#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and -#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per -#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET, -#: which is the mechanism the note above says survives the next table that is not an append table. -APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, - TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE}) - -#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. -#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger -#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram -#: column, an Instagram search — and they survive on the automations that already use them -#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). -#: -#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: -#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind -#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was -#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole -#: description: three machine nodes it does not have, and a one-line summary about searching -#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. -#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the -#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It -#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple -#: with no runner is a control that must refuse. -KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") -#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather -#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, -#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string -#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the -#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed -#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow -#: table resolving to the wrong default, and a summary sentence announcing the automation would -#: "do nothing yet". -#: -#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. -#: Three parallel string comparisons is precisely how the third platform gets missed twice more, -#: and the misses are individually invisible — each one degrades a different surface, none of them -#: raises, and every gate stays green (this whole family shipped green in wave 29). -#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, -#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a -#: corpus search?"* and nothing else; widening it into a platform registry would just move the -#: problem somewhere with a longer name. -DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") -#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT -#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so -#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry -#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and -#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about -#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned -#: a preset database under a dry-run fixture that asserts none exists. -#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a -#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one -#: agrees with law 1 rather than trusting that it does. -DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", - "tiktok_profile_match": "discover_tiktok"} -#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` -#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the -#: only things that differ are which connector answers and which tables the rows land in. -#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments -#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the -#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for -#: TikTok would start identical and drift, and the way it fails is that one network silently accepts -#: a limit the other refuses. -ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") -#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores -#: when the body names none, which after R6 is every create the client makes. -DEFAULT_KIND = "plain" -#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — -#: the ruling retires the door, not the two automations behind it (see `create`). -#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. -RETIRED_KINDS = ("scrape_db", "field_instagram") -#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were -#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name -#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence -#: per kind, in one place, because `create` and `clean_definition` both say it. -RETIRED_KIND_REPLACEMENT = { - "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", - "scrape_db": "Add a scrape step to any automation instead", -} -#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the -#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. -#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any -#: flow can take, so refusing the kind costs a person nothing but a different click. -#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared -#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a -#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch -#: door open on purpose, and there is a gate check that says so in those words. A kind may only be -#: walled off at a door once something else answers the same need. -#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) -#: ⭐ WAVE 34 · R12 — `plain` READS "Agent". The owner renamed the module to Agents, so the KIND -#: that means "an ordinary one of these" is an Agent, singular (the module is the plural). -#: ⛔ THE KEY `"plain"` IS UNTOUCHED and so is every other key here: D-65's rule, restated three -#: times in this file, is that a kind is permanent — the LABELS are display and may move freely, -#: the keys are stored on every automation ever created and may not. -KIND_LABELS = {"plain": "Agent", "scrape_db": "Web page to database", - "field_instagram": "Instagram profile column", - "discover_instagram": "Find Instagram profiles", - "discover_tiktok": "Find TikTok profiles"} -#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A SYSTEM AGENT CANNOT BE DELETED, one sentence per slug. -#: -#: ⛔ ONE PLACE, because two doors ask this question (the DELETE route refuses with it, and the -#: Canvas explains why its controls are read-only) and a refusal that says something different -#: from the surface it refuses on is worse than either sentence alone. D-65's own remedy in a new -#: family: a refusal that does not name the alternative sends somebody looking for a bug in a -#: decision made on purpose. -SYSTEM_AGENT_REASON = { - "field_agent": "this agent is an AI enrichment column. Delete the column on its database and " - "the agent goes with it", - "odoo_sync": "this is the Odoo connection's own sync schedule. Disconnect Odoo in Connectors " - "to stop it, or change how often it runs on this page", -} - - -def system_agent_refusal(slug): - """The sentence for a `system:` slug, or a general one for a slug nobody has written yet. - - ⚠ IT NEVER RETURNS EMPTY. A refusal with no sentence is a 409 a person cannot act on, and the - fallback is the one branch that will be reached by a slug added later and forgotten here. - """ - return SYSTEM_AGENT_REASON.get(str(slug or ""), "this agent is part of the workspace setup " - "and cannot be deleted here") - - -EXTRACTS = ("table", "jsonld") -STATES = ("idle", "running", "ok", "error", "partial") -#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). -#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a -#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). -TIERS = ("anonymous", "brightdata") -#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap -#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown -#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation -#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps -#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. -TIER_ALIASES = {"hiker": "brightdata"} - - -def clean_tier(raw): - """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" - t = str(raw or "").strip().lower() - t = TIER_ALIASES.get(t, t) - return t if t in TIERS else "" - - -def row_cap(table_key): - """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. - - An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything - else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. - """ - return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS - -#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI -#: shows and the vocabulary the parser accepts cannot drift. -CRON_PRESETS = [ - {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, - {"cron": "0 * * * *", "label": "Hourly"}, - {"cron": "0 6 * * *", "label": "Daily at 06:00"}, - {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, - {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, -] - -UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" - - -def _now(): - return _dt.datetime.now() - - -def _stamp(dt=None): - return (dt or _now()).strftime("%Y-%m-%d %H:%M") - - -def _iso(dt=None): - """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). - - ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables - and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A - naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC - minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is - not expressible at all. With the offset the same string is an instant, and relative rendering - becomes possible without migrating a single stored row. - - ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in - this module is against a naive `_now()`, and mixing aware and naive datetimes raises - `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. - """ - dt = dt or _now() - if dt.tzinfo is None: - dt = dt.astimezone() # a naive stamp from this process IS local time - return dt.isoformat(timespec="seconds") - - -#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. -#: -#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one -#: function down: `first_found` cells exist as post-D-18 offset stamps -#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s -#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A -#: converter that understood only the newest shape would blank the oldest rows — and a migration -#: that empties cells is indistinguishable from one that moved them. -#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" -#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and -#: shows up as itself instead of disappearing. -def _day(s): - """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" - raw = str(s or "").strip() - if not raw: - return "" - head = raw.replace("T", " ").split(" ")[0] - try: - _dt.date.fromisoformat(head) - except ValueError: - dt = _parse_iso(raw) - return dt.strftime("%Y-%m-%d") if dt else "" - return head - - -#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. -#: -#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` -#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every -#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse -#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 -#: dialect and a 0–100 dialect, and they meet here). -#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null -#: — and `0.0` would claim we measured zero engagement. -def _pct100(v): - """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" - if v is None or (isinstance(v, str) and not v.strip()): - return "" - try: - return _s(round(float(v) * 100.0, 4)) - except (TypeError, ValueError): - return "" - - -def _parse_iso(s): - """A stamp → a NAIVE LOCAL datetime, with or without an offset. - - Both shapes exist in the store simultaneously and always will: every run committed before - D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new - shape would silently return None for them — and `is_due` reads None as "never ran", which - would re-fire every schedule once. Reading both is what makes the change additive. - """ - raw = str(s or "").strip().replace("Z", "+00:00") - dt = None - try: - dt = _dt.datetime.fromisoformat(raw) - except Exception: # noqa: BLE001 - try: - dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") - except Exception: # noqa: BLE001 - return None - return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt - - -# --------------------------------------------------------------------------------------------- -# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher -# --------------------------------------------------------------------------------------------- -# The skill version guards the URL a developer typed. This one guards a URL that arrived in a -# request body, which changes the threat model in two ways the original does not cover: -# -# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` -# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` -# sails straight past a guard that only ever saw the first URL. The negative control ("a -# localhost URL is refused") would still pass while the rail was wide open. `fetch` below -# therefore takes the hops MANUALLY and re-guards every one. -# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the -# host and checks EVERY answer, not just the literal. -# -# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the -# record between the guard and the socket is not stopped by it. Closing that needs a pinned -# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather -# than implied away — the rail refuses the realistic cases and says what it does not cover. - -class Refused(ValueError): - """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" - - -def _ip_public(ip): - return not (ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_reserved or ip.is_unspecified or ip.is_multicast) - - -def guard(url, allowed=()): - """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers - checked too, plus the optional `--allowed` domain rail scrape.py carries.""" - p = urlparse(url or "") - if p.scheme not in ("http", "https"): - raise Refused(f"only http(s) allowed, not {p.scheme!r}") - host = (p.hostname or "").strip() - if not host: - raise Refused("no host in the URL") - low = host.lower() - if (low in ("localhost", "localhost.localdomain") - or low.endswith(".local") or low.endswith(".internal") - or low.endswith(".localhost")): - raise Refused(f"refused non-public host {host!r} (public web only)") - try: # a bare-IP host is decided on the literal - ip = ipaddress.ip_address(low) - except ValueError: - ip = None - if ip is not None and not _ip_public(ip): - raise Refused(f"refused non-public IP {host} (public web only)") - if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): - raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") - if ip is None: - try: - infos = socket.getaddrinfo(host, None) - except Exception as e: - raise Refused(f"could not resolve {host!r}: {type(e).__name__}") - for info in infos: - addr = info[4][0] - try: - resolved = ipaddress.ip_address(addr.split("%")[0]) - except ValueError: - continue - if not _ip_public(resolved): - raise Refused( - f"refused {host!r}: it resolves to the non-public address {resolved}") - return True - - -def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): - """GET `url`, taking redirects BY HAND so every hop passes `guard`. - - Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, - not a silent truncation — an endless redirect is indistinguishable from an attempt to walk - the fetcher somewhere it was told not to go. - """ - current = url - hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} - hdrs.update(headers or {}) - for _hop in range(max_hops + 1): - guard(current, allowed) - r = requests.get(current, timeout=timeout, headers=hdrs, - allow_redirects=False, stream=True) - if r.status_code in (301, 302, 303, 307, 308): - loc = r.headers.get("Location") - r.close() - if not loc: - raise Refused(f"redirect {r.status_code} with no Location header") - current = urljoin(current, loc) - continue - body = r.raw.read(max_kb * 1024, decode_content=True) or b"" - status, final = r.status_code, str(r.url) - r.close() - return status, final, body - raise Refused(f"more than {max_hops} redirects. Refusing to follow further") - - -def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): - """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. - - ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by - hand because a redirected GET is ordinary. A redirected POST is not: re-sending a - credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail - exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever - re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. - (The vendor calls below are the only POSTs this module makes, and they carry the API key.) - """ - guard(url) - hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} - hdrs.update(headers or {}) - r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, - data=json.dumps(body if body is not None else {}), stream=True) - try: - if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code}. A redirected POST is refused, " - f"never re-sent to a location the server picked") - return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") - finally: - r.close() - - -# --------------------------------------------------------------------------------------------- -# EXTRACTION — vendored from scrape.py, unchanged in behaviour -# --------------------------------------------------------------------------------------------- - -def _soup(body): - if BeautifulSoup is None: - raise RuntimeError( - "beautifulsoup4 is not installed in this environment. The automation engine's " - "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " - "aios-web/requirements.txt).") - try: - return BeautifulSoup(body, "lxml") - except Exception: - return BeautifulSoup(body, "html.parser") - - -def tables(soup): - """Every HTML table as rows. Dict rows when the first row looks like a header.""" - out = [] - for t in soup.find_all("table"): - rows = [] - for tr in t.find_all("tr"): - cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] - if cells: - rows.append(cells) - if not rows: - continue - head, body = rows[0], rows[1:] - if body and len(head) == len(body[0]) and all(head): - out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) - else: - out.append(rows) - return out - - -def jsonld(soup): - out = [] - for s in soup.find_all("script", attrs={"type": "application/ld+json"}): - try: - out.append(json.loads(s.string or s.get_text())) - except Exception: - pass - return out - - -def meta(soup): - m = {} - if soup.title and soup.title.string: - m["title"] = soup.title.string.strip() - for tag in soup.find_all("meta"): - k = tag.get("name") or tag.get("property") - v = tag.get("content") - if k and v and (k in ("description", "keywords", "author") - or k.startswith(("og:", "twitter:"))): - m[k] = v.strip() - return m - - -def preview(url, extract="table", table_index=0, allowed=()): - """The field-map preview the editor calls BEFORE anything is created: what columns does this - page actually offer, and what do the first rows look like? Never writes.""" - status, final, body = fetch(url, allowed=allowed) - if not (200 <= status < 300): - return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status}. It may be bot-gated or gone", - "columns": [], "sample": [], "rowCount": 0} - soup = _soup(body) - rows = [] - if extract == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 - picked = found[idx] if found else [] - rows = [r for r in picked if isinstance(r, dict)] - cols, seen = [], set() - for r in rows[:50]: - for k in r: - if k not in seen: - seen.add(k) - cols.append(k) - return {"ok": True, "status": status, "url": final, "columns": cols, - "sample": rows[:8], "rowCount": len(rows), - "tableCount": len(tables(soup)) if extract != "jsonld" else 0, - "title": (meta(soup) or {}).get("title", "")} - - -def _scalar(v): - if isinstance(v, (str, int, float)) and not isinstance(v, bool): - return str(v) - if isinstance(v, bool): - return "1" if v else "" - if isinstance(v, dict): - return str(v.get("name") or v.get("@id") or "") - if isinstance(v, list): - return ", ".join(_scalar(x) for x in v[:8]) - return "" - - -# --------------------------------------------------------------------------------------------- -# CRON — a 5-field parser and a "what was the last fire time" walk -# --------------------------------------------------------------------------------------------- -# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward -# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute -# between the last run and now?"* — and answering it that way is what makes a missed tick -# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and -# not never. - -_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] - - -def _parse_field(spec, lo, hi): - out = set() - for part in str(spec).split(","): - part = part.strip() - if not part: - raise ValueError("empty cron field part") - step = 1 - if "/" in part: - part, _, s = part.partition("/") - step = int(s) - if step < 1: - raise ValueError("cron step must be >= 1") - if part in ("*", "?"): - a, b = lo, hi - elif "-" in part.lstrip("-"): - a_s, _, b_s = part.partition("-") - a, b = int(a_s), int(b_s) - else: - a = b = int(part) - if a < lo or b > hi or a > b: - raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") - out.update(range(a, b + 1, step)) - return out - - -def parse_cron(expr): - """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. - - Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at - WRITE time, because a cron string nobody can evaluate is an automation that silently never - runs and reports no error anywhere. - """ - parts = str(expr or "").split() - if len(parts) != 5: - raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") - sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] - dom_r = parts[2].strip() not in ("*", "?") - dow_r = parts[4].strip() not in ("*", "?") - return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), - dom_r, dow_r) - - -def _day_matches(day, doms, months, dows, dom_r, dow_r): - if day.month not in months: - return False - # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the - # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes - # a schedule that looks right fire almost never. - dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 - if dom_r and dow_r: - return day.day in doms or dow in dows - if dom_r: - return day.day in doms - if dow_r: - return dow in dows - return True - - -def prev_fire(expr, now=None, lookback_days=400): - """The most recent scheduled minute at or before `now`, or None inside the lookback. - - Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, - and only a matching day needs its hours/minutes searched. - """ - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - now = (now or _now()).replace(second=0, microsecond=0) - for back in range(lookback_days + 1): - day = (now - _dt.timedelta(days=back)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = back == 0 - for h in sorted(hours, reverse=True): - if same_day and h > now.hour: - continue - for m in sorted(minutes, reverse=True): - if same_day and h == now.hour and m > now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -def is_due(defn, now=None): - """Should the scheduler run this automation right now? - - Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the - moment the schedule was enabled, else creation) and at or before now. Anchoring on - `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at - 14:00" from running immediately and looking like a bug. - """ - if not isinstance(defn, dict): - return False - sched = defn.get("schedule") or {} - if not sched.get("enabled"): - return False - try: - fire = prev_fire(sched.get("cron"), now=now) - except ValueError: - return False - if fire is None: - return False - since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) - or _parse_iso(sched.get("enabledAt")) - or _parse_iso(defn.get("created"))) - return since is None or fire > since - - -def next_fire(expr, now=None, lookahead_days=400): - """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" - try: - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - except ValueError: - return None - now = (now or _now()).replace(second=0, microsecond=0) - for ahead in range(lookahead_days + 1): - day = (now + _dt.timedelta(days=ahead)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = ahead == 0 - for h in sorted(hours): - if same_day and h < now.hour: - continue - for m in sorted(minutes): - if same_day and h == now.hour and m <= now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -# --------------------------------------------------------------------------------------------- -# THE UPSERT — pure, so the arithmetic is testable without a store or a network -# --------------------------------------------------------------------------------------------- - -#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. -#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled -#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by -#: pre-collected corpus values of unknown vintage — no error, no visible change other than the -#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this -#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. -#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus -#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read -#: belongs. Suppressing the observation would be the same defect arriving from the other side. -CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") - - -def corpus_protect(before, _src): - """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. - - ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell - non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making - `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A - re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read - owns. - """ - return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () - - -def upsert_rows(existing, incoming, key_field, cap=None, protect=None): - """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. - - `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row - may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are - untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to - the merge, and a rule baked in here would apply to nine paths that never asked for one. - - THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped - appearing on the source page has not necessarily stopped existing — the page changed its - filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream - hiccup into silent data loss, and the row may be carrying user-typed overlay values in - columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. - - Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. - - ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on - entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to - call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` - that same loop is hundreds of millions of dict copies and the automation simply never - finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) - - ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different - facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the - DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — - a property of the SYSTEM, and never benign. One number for both meant a table hitting its - ceiling was indistinguishable from a page with a few blank cells, which is how a time series - stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. - """ - cap = MAX_UT_ROWS if cap is None else int(cap) - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, - "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} - by_key, dupe_ids = {}, set() - for rid, row in rows.items(): - kv = str(row.get(key_field, "") or "").strip() - if not kv: - continue - if kv in by_key: - dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left - continue - by_key[kv] = rid - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 - seen_keys, incoming_dupes = set(), 0 - for src in incoming or []: - kv = str((src or {}).get(key_field, "") or "").strip() - if not kv: - counts["skipped"] += 1 # no key -> cannot be upserted; never guessed - continue - if kv in seen_keys: - incoming_dupes += 1 # the SOURCE listed it twice; first wins - continue - seen_keys.add(kv) - rid = by_key.get(kv) - if rid is None: - if len(rows) >= cap: - counts["capped"] += 1 # LOUD: the runner turns this into a partial run - continue - rid = str(next_id) - next_id += 1 - rows[rid] = dict(src) - by_key[kv] = rid - counts["inserted"] += 1 - continue - before = rows[rid] - # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. - # `held` counts the cells a lower-provenance source was refused, so the run can SAY it - # rather than quietly doing the right thing — a protection nobody is told about is - # indistinguishable from a source that happened to agree. - keep = set(protect(before, src) or ()) if protect else set() - use = {k: v for k, v in src.items() if k not in keep} if keep else src - if keep: - counts["held"] = counts.get("held", 0) + sum( - 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) - changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} - if changed: - before.update(use) - counts["updated"] += 1 - else: - counts["unchanged"] += 1 - counts["duplicates"] = incoming_dupes + len(dupe_ids) - counts["orphans"] = sum( - 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) - return rows, counts - - -def dedupe_canonical_rows(existing, key_field, newest_by=""): - """Collapse duplicate logical rows while preserving the lowest stable row id. - - Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: - repeated shortcodes there are new timestamped observations, not duplicate posts. Values are - taken newest-first and then filled from older rows, so a sparse fresh projection does not - erase a field an earlier row knew. - """ - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - groups = {} - for rid, row in rows.items(): - identity = str(row.get(key_field) or "").strip() - if identity: - groups.setdefault(identity, []).append((rid, row)) - removed = 0 - for members in groups.values(): - if len(members) < 2: - continue - keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) - ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ - if newest_by else members - merged = {} - for _rid, row in ordered: - for key, value in row.items(): - if key not in merged or str(merged.get(key) or "").strip() == "": - merged[key] = value - rows[keep] = merged - for rid, _row in members: - if rid != keep: - rows.pop(rid, None) - removed += 1 - return rows, removed - - -# --------------------------------------------------------------------------------------------- -# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) -# --------------------------------------------------------------------------------------------- - -def _ut_slug(label): - s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return (s or "table")[:40] - - -def ut_all(rt): - try: - return dict(rt.get(UT_STORE_KEY) or {}) - except Exception: - return {} - - -def disable_for_table(rt, table_key, note="target database deleted"): - """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. - - The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still - shows what existed and why it stopped — silently deleting a user's automation because its - target died would read as data loss. Returns the ids it touched.""" - key = str(table_key or "") - touched = [] - - def _up(cur): - for aid, d in (cur or {}).items(): - if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: - sch = d.get("schedule") +"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. + +THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant +store and a RUN that lives in this process. The definition is durable, small and rarely written; +the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file +is built to avoid, and it has two halves: + + * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the + store outlives the process that wrote it, so a container restart mid-run leaves an automation + that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives + in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written + exactly ONCE per run, at the end. + * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s + floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape + by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes + its whole result first and commits it in a single `update` callback. + +WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not +deployed, so its extraction core is copied into this file (the wave doc's instruction), ported +httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with +it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: +the skill version takes a URL from a developer on a CLI; this takes one from a request body. +""" +from __future__ import annotations + +import copy +import datetime as _dt +import hashlib +import hmac +import ipaddress +import json +import os +import re +import socket +import threading +import time +from urllib.parse import urljoin, urlparse + +import requests + +# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability +# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — +# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the +# connector that routes, which is the shape the split was for: this file no longer knows that +# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore +# a signal, not a convenience — it means something in the automation runtime started making a +# vendor decision, and that belongs one layer down. + +try: # the parser the extraction half needs + from bs4 import BeautifulSoup +except Exception: # pragma: no cover — deploy lag; see mailbox ② + BeautifulSoup = None + +# --------------------------------------------------------------------------------------------- +# THE BUCKET (C4-AUTO) +# --------------------------------------------------------------------------------------------- + +#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. +STORE_KEY = "automations" +#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through +#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for +#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the +#: session-D mailbox as a finding against A's file rather than fixed from here. +UT_STORE_KEY = "user_tables" +UT_PREFIX = "ut_" + +MAX_AUTOMATIONS = 40 +MAX_RUNS = 20 # trimmed history per automation +MAX_NAME = 80 +#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x +#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it +#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently +#: walking 5,000 of 32,826 connected rows. It was. +#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from +#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no +#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is +#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. +#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing +#: should not also be the widest one. And do not delete it — D-143's own row says the editable +#: substrate still needs a bound. +MAX_UT_ROWS = 5000 +MAX_UT_TABLES = 40 + +#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. +#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; +#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real +#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — +#: indistinguishable on every surface from a run that worked, and a strong candidate for why the +#: owner's enrichment kept appearing to do nothing. +#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better +#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. +ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" + +#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. +#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the +#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull +#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's +#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour +#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still +#: report success. A time series that stops after five days without saying so is worse than one +#: that was never built. +#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE +#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: +#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day +#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts +#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had +#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the +#: silent version would have flatlined a chart inside a fortnight with a green dot over it. +#: +#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises +#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, +#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 +#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a +#: smaller number. +#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) +MAX_UT_IG_ROWS = 200_000 +IG_TABLE_PREFIX = "ut_ig_" + +#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was +#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — +#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY +#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would +#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket +#: serialisation cost, purely by an accident of naming. Naming the append tables is the version +#: that stays true when the next `ut_ig_*` table is not one. +IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" +IG_POSTS_TABLE = "ut_ig_posts" +IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" +IG_COMMENTS_TABLE = "ut_ig_comments" + +# --------------------------------------------------------------------------------------------- +# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY +# --------------------------------------------------------------------------------------------- +# R1: FULL PARITY — profile AND posts AND comments, not a profile tier. +# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live +# Instagram rows, and the two schemas may DIVERGE where the vendors do. +# +# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is +# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity +# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`. +# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on +# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video +# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables +# holding hundreds of thousands of rows is a migration on production data to buy a shared grid +# nobody asked for. R2 chose the version with no migration. +# +# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn +# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST +# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says +# `tt_id` rather than inheriting a column labelled "Instagram id". +TT_TABLE_PREFIX = "ut_tt_" +TT_PROFILE_TABLE = "ut_tt_profile" +TT_SNAPSHOTS_TABLE = "ut_tt_snapshots" +TT_POSTS_TABLE = "ut_tt_posts" +TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots" +TT_COMMENTS_TABLE = "ut_tt_comments" + +AUTOMATION_RECORD_MODE = "automation" +#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and +#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per +#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET, +#: which is the mechanism the note above says survives the next table that is not an append table. +APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, + TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE}) + +#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. +#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger +#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram +#: column, an Instagram search — and they survive on the automations that already use them +#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). +#: +#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: +#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind +#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was +#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole +#: description: three machine nodes it does not have, and a one-line summary about searching +#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. +#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the +#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It +#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple +#: with no runner is a control that must refuse. +KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") +#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather +#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, +#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string +#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the +#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed +#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow +#: table resolving to the wrong default, and a summary sentence announcing the automation would +#: "do nothing yet". +#: +#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. +#: Three parallel string comparisons is precisely how the third platform gets missed twice more, +#: and the misses are individually invisible — each one degrades a different surface, none of them +#: raises, and every gate stays green (this whole family shipped green in wave 29). +#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, +#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a +#: corpus search?"* and nothing else; widening it into a platform registry would just move the +#: problem somewhere with a longer name. +DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") +#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT +#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so +#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry +#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and +#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about +#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned +#: a preset database under a dry-run fixture that asserts none exists. +#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a +#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one +#: agrees with law 1 rather than trusting that it does. +DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", + "tiktok_profile_match": "discover_tiktok"} +#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` +#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the +#: only things that differ are which connector answers and which tables the rows land in. +#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments +#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the +#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for +#: TikTok would start identical and drift, and the way it fails is that one network silently accepts +#: a limit the other refuses. +ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") +#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores +#: when the body names none, which after R6 is every create the client makes. +DEFAULT_KIND = "plain" +#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — +#: the ruling retires the door, not the two automations behind it (see `create`). +#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. +RETIRED_KINDS = ("scrape_db", "field_instagram") +#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were +#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name +#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence +#: per kind, in one place, because `create` and `clean_definition` both say it. +RETIRED_KIND_REPLACEMENT = { + "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", + "scrape_db": "Add a scrape step to any automation instead", +} +#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the +#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. +#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any +#: flow can take, so refusing the kind costs a person nothing but a different click. +#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared +#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a +#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch +#: door open on purpose, and there is a gate check that says so in those words. A kind may only be +#: walled off at a door once something else answers the same need. +#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) +#: ⭐ WAVE 34 · R12 — `plain` READS "Agent". The owner renamed the module to Agents, so the KIND +#: that means "an ordinary one of these" is an Agent, singular (the module is the plural). +#: ⛔ THE KEY `"plain"` IS UNTOUCHED and so is every other key here: D-65's rule, restated three +#: times in this file, is that a kind is permanent — the LABELS are display and may move freely, +#: the keys are stored on every automation ever created and may not. +KIND_LABELS = {"plain": "Agent", "scrape_db": "Web page to database", + "field_instagram": "Instagram profile column", + "discover_instagram": "Find Instagram profiles", + "discover_tiktok": "Find TikTok profiles"} +#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A SYSTEM AGENT CANNOT BE DELETED, one sentence per slug. +#: +#: ⛔ ONE PLACE, because two doors ask this question (the DELETE route refuses with it, and the +#: Canvas explains why its controls are read-only) and a refusal that says something different +#: from the surface it refuses on is worse than either sentence alone. D-65's own remedy in a new +#: family: a refusal that does not name the alternative sends somebody looking for a bug in a +#: decision made on purpose. +SYSTEM_AGENT_REASON = { + "field_agent": "this agent is an AI enrichment column. Delete the column on its database and " + "the agent goes with it", + "odoo_sync": "this is the Odoo connection's own sync schedule. Disconnect Odoo in Connectors " + "to stop it, or change how often it runs on this page", +} + + +def system_agent_refusal(slug): + """The sentence for a `system:` slug, or a general one for a slug nobody has written yet. + + ⚠ IT NEVER RETURNS EMPTY. A refusal with no sentence is a 409 a person cannot act on, and the + fallback is the one branch that will be reached by a slug added later and forgotten here. + """ + return SYSTEM_AGENT_REASON.get(str(slug or ""), "this agent is part of the workspace setup " + "and cannot be deleted here") + + +EXTRACTS = ("table", "jsonld") +STATES = ("idle", "running", "ok", "error", "partial") +#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). +#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a +#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). +TIERS = ("anonymous", "brightdata") +#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap +#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown +#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation +#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps +#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. +TIER_ALIASES = {"hiker": "brightdata"} + + +def clean_tier(raw): + """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" + t = str(raw or "").strip().lower() + t = TIER_ALIASES.get(t, t) + return t if t in TIERS else "" + + +def row_cap(table_key): + """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. + + An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything + else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. + """ + return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS + +#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI +#: shows and the vocabulary the parser accepts cannot drift. +CRON_PRESETS = [ + {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, + {"cron": "0 * * * *", "label": "Hourly"}, + {"cron": "0 6 * * *", "label": "Daily at 06:00"}, + {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, + {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, +] + +UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" + + +def _now(): + return _dt.datetime.now() + + +def _stamp(dt=None): + return (dt or _now()).strftime("%Y-%m-%d %H:%M") + + +def _iso(dt=None): + """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). + + ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables + and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A + naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC + minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is + not expressible at all. With the offset the same string is an instant, and relative rendering + becomes possible without migrating a single stored row. + + ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in + this module is against a naive `_now()`, and mixing aware and naive datetimes raises + `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. + """ + dt = dt or _now() + if dt.tzinfo is None: + dt = dt.astimezone() # a naive stamp from this process IS local time + return dt.isoformat(timespec="seconds") + + +#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. +#: +#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one +#: function down: `first_found` cells exist as post-D-18 offset stamps +#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s +#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A +#: converter that understood only the newest shape would blank the oldest rows — and a migration +#: that empties cells is indistinguishable from one that moved them. +#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" +#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and +#: shows up as itself instead of disappearing. +def _day(s): + """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" + raw = str(s or "").strip() + if not raw: + return "" + head = raw.replace("T", " ").split(" ")[0] + try: + _dt.date.fromisoformat(head) + except ValueError: + dt = _parse_iso(raw) + return dt.strftime("%Y-%m-%d") if dt else "" + return head + + +#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. +#: +#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` +#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every +#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse +#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 +#: dialect and a 0–100 dialect, and they meet here). +#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null +#: — and `0.0` would claim we measured zero engagement. +def _pct100(v): + """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" + if v is None or (isinstance(v, str) and not v.strip()): + return "" + try: + return _s(round(float(v) * 100.0, 4)) + except (TypeError, ValueError): + return "" + + +def _parse_iso(s): + """A stamp → a NAIVE LOCAL datetime, with or without an offset. + + Both shapes exist in the store simultaneously and always will: every run committed before + D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new + shape would silently return None for them — and `is_due` reads None as "never ran", which + would re-fire every schedule once. Reading both is what makes the change additive. + """ + raw = str(s or "").strip().replace("Z", "+00:00") + dt = None + try: + dt = _dt.datetime.fromisoformat(raw) + except Exception: # noqa: BLE001 + try: + dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") + except Exception: # noqa: BLE001 + return None + return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt + + +# --------------------------------------------------------------------------------------------- +# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher +# --------------------------------------------------------------------------------------------- +# The skill version guards the URL a developer typed. This one guards a URL that arrived in a +# request body, which changes the threat model in two ways the original does not cover: +# +# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` +# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` +# sails straight past a guard that only ever saw the first URL. The negative control ("a +# localhost URL is refused") would still pass while the rail was wide open. `fetch` below +# therefore takes the hops MANUALLY and re-guards every one. +# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the +# host and checks EVERY answer, not just the literal. +# +# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the +# record between the guard and the socket is not stopped by it. Closing that needs a pinned +# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather +# than implied away — the rail refuses the realistic cases and says what it does not cover. + +class Refused(ValueError): + """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" + + +def _ip_public(ip): + return not (ip.is_private or ip.is_loopback or ip.is_link_local + or ip.is_reserved or ip.is_unspecified or ip.is_multicast) + + +def guard(url, allowed=()): + """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers + checked too, plus the optional `--allowed` domain rail scrape.py carries.""" + p = urlparse(url or "") + if p.scheme not in ("http", "https"): + raise Refused(f"only http(s) allowed, not {p.scheme!r}") + host = (p.hostname or "").strip() + if not host: + raise Refused("no host in the URL") + low = host.lower() + if (low in ("localhost", "localhost.localdomain") + or low.endswith(".local") or low.endswith(".internal") + or low.endswith(".localhost")): + raise Refused(f"refused non-public host {host!r} (public web only)") + try: # a bare-IP host is decided on the literal + ip = ipaddress.ip_address(low) + except ValueError: + ip = None + if ip is not None and not _ip_public(ip): + raise Refused(f"refused non-public IP {host} (public web only)") + if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): + raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") + if ip is None: + try: + infos = socket.getaddrinfo(host, None) + except Exception as e: + raise Refused(f"could not resolve {host!r}: {type(e).__name__}") + for info in infos: + addr = info[4][0] + try: + resolved = ipaddress.ip_address(addr.split("%")[0]) + except ValueError: + continue + if not _ip_public(resolved): + raise Refused( + f"refused {host!r}: it resolves to the non-public address {resolved}") + return True + + +def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): + """GET `url`, taking redirects BY HAND so every hop passes `guard`. + + Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, + not a silent truncation — an endless redirect is indistinguishable from an attempt to walk + the fetcher somewhere it was told not to go. + """ + current = url + hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} + hdrs.update(headers or {}) + for _hop in range(max_hops + 1): + guard(current, allowed) + r = requests.get(current, timeout=timeout, headers=hdrs, + allow_redirects=False, stream=True) + if r.status_code in (301, 302, 303, 307, 308): + loc = r.headers.get("Location") + r.close() + if not loc: + raise Refused(f"redirect {r.status_code} with no Location header") + current = urljoin(current, loc) + continue + body = r.raw.read(max_kb * 1024, decode_content=True) or b"" + status, final = r.status_code, str(r.url) + r.close() + return status, final, body + raise Refused(f"more than {max_hops} redirects. Refusing to follow further") + + +def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): + """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. + + ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by + hand because a redirected GET is ordinary. A redirected POST is not: re-sending a + credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail + exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever + re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. + (The vendor calls below are the only POSTs this module makes, and they carry the API key.) + """ + guard(url) + hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} + hdrs.update(headers or {}) + r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, + data=json.dumps(body if body is not None else {}), stream=True) + try: + if r.status_code in (301, 302, 303, 307, 308): + raise Refused(f"the POST answered {r.status_code}. A redirected POST is refused, " + f"never re-sent to a location the server picked") + return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") + finally: + r.close() + + +# --------------------------------------------------------------------------------------------- +# EXTRACTION — vendored from scrape.py, unchanged in behaviour +# --------------------------------------------------------------------------------------------- + +def _soup(body): + if BeautifulSoup is None: + raise RuntimeError( + "beautifulsoup4 is not installed in this environment. The automation engine's " + "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " + "aios-web/requirements.txt).") + try: + return BeautifulSoup(body, "lxml") + except Exception: + return BeautifulSoup(body, "html.parser") + + +def tables(soup): + """Every HTML table as rows. Dict rows when the first row looks like a header.""" + out = [] + for t in soup.find_all("table"): + rows = [] + for tr in t.find_all("tr"): + cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] + if cells: + rows.append(cells) + if not rows: + continue + head, body = rows[0], rows[1:] + if body and len(head) == len(body[0]) and all(head): + out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) + else: + out.append(rows) + return out + + +def jsonld(soup): + out = [] + for s in soup.find_all("script", attrs={"type": "application/ld+json"}): + try: + out.append(json.loads(s.string or s.get_text())) + except Exception: + pass + return out + + +def meta(soup): + m = {} + if soup.title and soup.title.string: + m["title"] = soup.title.string.strip() + for tag in soup.find_all("meta"): + k = tag.get("name") or tag.get("property") + v = tag.get("content") + if k and v and (k in ("description", "keywords", "author") + or k.startswith(("og:", "twitter:"))): + m[k] = v.strip() + return m + + +def preview(url, extract="table", table_index=0, allowed=()): + """The field-map preview the editor calls BEFORE anything is created: what columns does this + page actually offer, and what do the first rows look like? Never writes.""" + status, final, body = fetch(url, allowed=allowed) + if not (200 <= status < 300): + return {"ok": False, "status": status, "url": final, + "note": f"the page answered {status}. It may be bot-gated or gone", + "columns": [], "sample": [], "rowCount": 0} + soup = _soup(body) + rows = [] + if extract == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 + picked = found[idx] if found else [] + rows = [r for r in picked if isinstance(r, dict)] + cols, seen = [], set() + for r in rows[:50]: + for k in r: + if k not in seen: + seen.add(k) + cols.append(k) + return {"ok": True, "status": status, "url": final, "columns": cols, + "sample": rows[:8], "rowCount": len(rows), + "tableCount": len(tables(soup)) if extract != "jsonld" else 0, + "title": (meta(soup) or {}).get("title", "")} + + +def _scalar(v): + if isinstance(v, (str, int, float)) and not isinstance(v, bool): + return str(v) + if isinstance(v, bool): + return "1" if v else "" + if isinstance(v, dict): + return str(v.get("name") or v.get("@id") or "") + if isinstance(v, list): + return ", ".join(_scalar(x) for x in v[:8]) + return "" + + +# --------------------------------------------------------------------------------------------- +# CRON — a 5-field parser and a "what was the last fire time" walk +# --------------------------------------------------------------------------------------------- +# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward +# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute +# between the last run and now?"* — and answering it that way is what makes a missed tick +# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and +# not never. + +_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] + + +def _parse_field(spec, lo, hi): + out = set() + for part in str(spec).split(","): + part = part.strip() + if not part: + raise ValueError("empty cron field part") + step = 1 + if "/" in part: + part, _, s = part.partition("/") + step = int(s) + if step < 1: + raise ValueError("cron step must be >= 1") + if part in ("*", "?"): + a, b = lo, hi + elif "-" in part.lstrip("-"): + a_s, _, b_s = part.partition("-") + a, b = int(a_s), int(b_s) + else: + a = b = int(part) + if a < lo or b > hi or a > b: + raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") + out.update(range(a, b + 1, step)) + return out + + +def parse_cron(expr): + """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. + + Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at + WRITE time, because a cron string nobody can evaluate is an automation that silently never + runs and reports no error anywhere. + """ + parts = str(expr or "").split() + if len(parts) != 5: + raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") + sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] + dom_r = parts[2].strip() not in ("*", "?") + dow_r = parts[4].strip() not in ("*", "?") + return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), + dom_r, dow_r) + + +def _day_matches(day, doms, months, dows, dom_r, dow_r): + if day.month not in months: + return False + # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the + # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes + # a schedule that looks right fire almost never. + dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 + if dom_r and dow_r: + return day.day in doms or dow in dows + if dom_r: + return day.day in doms + if dow_r: + return dow in dows + return True + + +def prev_fire(expr, now=None, lookback_days=400): + """The most recent scheduled minute at or before `now`, or None inside the lookback. + + Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, + and only a matching day needs its hours/minutes searched. + """ + minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) + now = (now or _now()).replace(second=0, microsecond=0) + for back in range(lookback_days + 1): + day = (now - _dt.timedelta(days=back)).date() + if not _day_matches(day, doms, months, dows, dom_r, dow_r): + continue + same_day = back == 0 + for h in sorted(hours, reverse=True): + if same_day and h > now.hour: + continue + for m in sorted(minutes, reverse=True): + if same_day and h == now.hour and m > now.minute: + continue + return _dt.datetime(day.year, day.month, day.day, h, m) + return None + + +def is_due(defn, now=None): + """Should the scheduler run this automation right now? + + Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the + moment the schedule was enabled, else creation) and at or before now. Anchoring on + `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at + 14:00" from running immediately and looking like a bug. + """ + if not isinstance(defn, dict): + return False + sched = defn.get("schedule") or {} + if not sched.get("enabled"): + return False + try: + fire = prev_fire(sched.get("cron"), now=now) + except ValueError: + return False + if fire is None: + return False + since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) + or _parse_iso(sched.get("enabledAt")) + or _parse_iso(defn.get("created"))) + return since is None or fire > since + + +def next_fire(expr, now=None, lookahead_days=400): + """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" + try: + minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) + except ValueError: + return None + now = (now or _now()).replace(second=0, microsecond=0) + for ahead in range(lookahead_days + 1): + day = (now + _dt.timedelta(days=ahead)).date() + if not _day_matches(day, doms, months, dows, dom_r, dow_r): + continue + same_day = ahead == 0 + for h in sorted(hours): + if same_day and h < now.hour: + continue + for m in sorted(minutes): + if same_day and h == now.hour and m <= now.minute: + continue + return _dt.datetime(day.year, day.month, day.day, h, m) + return None + + +# --------------------------------------------------------------------------------------------- +# THE UPSERT — pure, so the arithmetic is testable without a store or a network +# --------------------------------------------------------------------------------------------- + +#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. +#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled +#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by +#: pre-collected corpus values of unknown vintage — no error, no visible change other than the +#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this +#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. +#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus +#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read +#: belongs. Suppressing the observation would be the same defect arriving from the other side. +CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") + + +def corpus_protect(before, _src): + """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. + + ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell + non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making + `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A + re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read + owns. + """ + return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () + + +def upsert_rows(existing, incoming, key_field, cap=None, protect=None): + """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. + + `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row + may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are + untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to + the merge, and a rule baked in here would apply to nine paths that never asked for one. + + THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped + appearing on the source page has not necessarily stopped existing — the page changed its + filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream + hiccup into silent data loss, and the row may be carrying user-typed overlay values in + columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. + + Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. + + ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on + entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to + call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` + that same loop is hundreds of millions of dict copies and the automation simply never + finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) + + ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different + facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the + DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — + a property of the SYSTEM, and never benign. One number for both meant a table hitting its + ceiling was indistinguishable from a page with a few blank cells, which is how a time series + stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. + """ + cap = MAX_UT_ROWS if cap is None else int(cap) + rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} + counts = {"inserted": 0, "updated": 0, "unchanged": 0, + "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} + by_key, dupe_ids = {}, set() + for rid, row in rows.items(): + kv = str(row.get(key_field, "") or "").strip() + if not kv: + continue + if kv in by_key: + dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left + continue + by_key[kv] = rid + next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 + seen_keys, incoming_dupes = set(), 0 + for src in incoming or []: + kv = str((src or {}).get(key_field, "") or "").strip() + if not kv: + counts["skipped"] += 1 # no key -> cannot be upserted; never guessed + continue + if kv in seen_keys: + incoming_dupes += 1 # the SOURCE listed it twice; first wins + continue + seen_keys.add(kv) + rid = by_key.get(kv) + if rid is None: + if len(rows) >= cap: + counts["capped"] += 1 # LOUD: the runner turns this into a partial run + continue + rid = str(next_id) + next_id += 1 + rows[rid] = dict(src) + by_key[kv] = rid + counts["inserted"] += 1 + continue + before = rows[rid] + # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. + # `held` counts the cells a lower-provenance source was refused, so the run can SAY it + # rather than quietly doing the right thing — a protection nobody is told about is + # indistinguishable from a source that happened to agree. + keep = set(protect(before, src) or ()) if protect else set() + use = {k: v for k, v in src.items() if k not in keep} if keep else src + if keep: + counts["held"] = counts.get("held", 0) + sum( + 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) + changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} + if changed: + before.update(use) + counts["updated"] += 1 + else: + counts["unchanged"] += 1 + counts["duplicates"] = incoming_dupes + len(dupe_ids) + counts["orphans"] = sum( + 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) + return rows, counts + + +def dedupe_canonical_rows(existing, key_field, newest_by=""): + """Collapse duplicate logical rows while preserving the lowest stable row id. + + Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: + repeated shortcodes there are new timestamped observations, not duplicate posts. Values are + taken newest-first and then filled from older rows, so a sparse fresh projection does not + erase a field an earlier row knew. + """ + rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} + groups = {} + for rid, row in rows.items(): + identity = str(row.get(key_field) or "").strip() + if identity: + groups.setdefault(identity, []).append((rid, row)) + removed = 0 + for members in groups.values(): + if len(members) < 2: + continue + keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) + ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ + if newest_by else members + merged = {} + for _rid, row in ordered: + for key, value in row.items(): + if key not in merged or str(merged.get(key) or "").strip() == "": + merged[key] = value + rows[keep] = merged + for rid, _row in members: + if rid != keep: + rows.pop(rid, None) + removed += 1 + return rows, removed + + +# --------------------------------------------------------------------------------------------- +# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) +# --------------------------------------------------------------------------------------------- + +def _ut_slug(label): + s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") + return (s or "table")[:40] + + +def ut_all(rt): + try: + return dict(rt.get(UT_STORE_KEY) or {}) + except Exception: + return {} + + +def disable_for_table(rt, table_key, note="target database deleted"): + """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. + + The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still + shows what existed and why it stopped — silently deleting a user's automation because its + target died would read as data loss. Returns the ids it touched.""" + key = str(table_key or "") + touched = [] + + def _up(cur): + for aid, d in (cur or {}).items(): + if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: + sch = d.get("schedule") if not isinstance(sch, dict): sch = d["schedule"] = {} sch["enabled"] = False @@ -963,12355 +963,12727 @@ def disable_for_table(rt, table_key, note="target database deleted"): if isinstance(trg, dict): trg["paused"] = True d["statusNote"] = note - touched.append(str(aid)) - return cur - - if key: - _store_update(rt, _up, flush="sync") - return touched - - -def ut_get(rt, key): - return ut_all(rt).get(str(key)) - - -def retire_automation_stage_fields(rt, tables=None): - """Delete obsolete Board-only fields and their hidden cells, never user fields. - - The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` - metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old - generated timestamp/cycle cells are cleared too, including rows whose field definition was - removed by an interrupted earlier migration. Re-running this migration is a no-op. - - ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is - O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to - deep-copy (see this module's header), and `GET /automations` was paying for THREE independent - copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the - WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent - snapshot can never be the thing that gets written back. - """ - tables = ut_all(rt) if tables is None else tables - stage_keys = set() - for table in tables.values(): - if not isinstance(table, dict): - continue - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): - stage_keys.add(str(field.get("key") or "")) - for row in (table.get("rows") or {}).values(): - for key in (row or {}): - text = str(key) - if text.startswith("stage_auto_"): - stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) - stage_keys.discard("") - if not stage_keys: - return {"tables": 0, "fields": 0, "cells": 0} - - changed = {"tables": set(), "fields": 0, "cells": 0} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for table_key, table in cur.items(): - if not isinstance(table, dict): - continue - kept, removed = [], set() - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - key = str(field.get("key") or "") if isinstance(field, dict) else "" - if key in stage_keys and isinstance(auto, dict) and \ - (auto.get("stageField") or auto.get("cyclesField")): - removed.add(key) - changed["fields"] += 1 - continue - kept.append(field) - if removed: - table["fields"] = kept - changed["tables"].add(str(table_key)) - for row in (table.get("rows") or {}).values(): - if not isinstance(row, dict): - continue - for stage_key in stage_keys: - for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): - if key in row: - row.pop(key, None) - changed["cells"] += 1 - changed["tables"].add(str(table_key)) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return {"tables": len(changed["tables"]), "fields": changed["fields"], - "cells": changed["cells"]} - - -def bind_unbound_fields(rt): - """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` - are bound to the definition that already writes them — matched on the (targetTable, - fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no - definition references is DISABLED with the reason on it rather than deleted or guessed: - the column was already dead (nothing runs a column no flow names), and now it says so. - Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" - by_binding = {} - for aid, d in all_definitions(rt).items(): - cfg = d.get("config") or {} - if cfg.get("fieldKey") and cfg.get("targetTable"): - by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) - plan = {} - for tk, t in ut_all(rt).items(): - for f in (t.get("fields") or []): - a = f.get("automation") - # An already-DISABLED bag is a decision this migration made on a previous pass — - # re-planning it every call would turn the once-per-process sweep into a write - # per read. - if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ - and not a.get("disabled"): - plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) - if not plan: - return 0, 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for (tk, fk), aid in plan.items(): - for f in ((cur.get(tk) or {}).get("fields") or []): - if f.get("key") == fk and isinstance(f.get("automation"), dict): - if aid: - f["automation"]["flowId"] = aid - else: - f["automation"]["disabled"] = True - f["automation"]["statusNote"] = ( - "not bound to any flow. Create an automation for this column " - "or delete it") - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - bound = sum(1 for v in plan.values() if v) - return bound, len(plan) - bound - - -def ut_key_for(label, key=None): - """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which - must not create anything) resolves the same key by construction rather than by copying the - derivation and drifting from it.""" - k = str(key or (UT_PREFIX + _ut_slug(label))) - return k if k.startswith(UT_PREFIX) else UT_PREFIX + k - - -#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same -#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging -#: `core` into the API's boot path — and the two are held in step by a gate check. -MACHINE_OWNERS = ("automation", "scheduler") - - -def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", - record_mode="", lock_fields=False, tables=None): - """Create the table if it is missing; return its key. Idempotent — a re-run of an automation - that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) - and an existing table with that key is adopted, not duplicated. - - ⭐⭐ WAVE 31 · T30 — `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is - the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every - call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass - paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still - takes forever to change the Config for automation as well. It just say Saving..."* — measured - **7,912 ms** for one save on `4258a93`. - - ⚠ THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope — the same argument - `retire_automation_stage_fields` (this module, same parameter name, same contract) already - makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`, - whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore - never be the thing that gets written back**, and the worst a stale one can do is spend a commit - that would have been skipped — never write a wrong value. Callers that ensure two tables - under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`). - - Fields are MERGED, never replaced: a user who added a column to an automation's table keeps - it, and a new source column joins on the next run. - - `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped - `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their - provenance. Fields already on the table keep whatever tag they have (first flow wins; - shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not - ownership). - - ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever - or WHATEVER ran the automation, so the same table belonged to a person if its first run was - manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` - admits only the creator or an admin, so **whether you could open your own Instagram - snapshots depended on a race you never saw**. The owner is now the automation's creator, and - a table already stamped with a machine name is ADOPTED the next time a run knows a human - one. Adoption is a repair, not a widening: the automation's creator is the person who asked - for the table in the first place. - """ - key = ut_key_for(label, key) - # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. - # - # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it - # would leave every table already in production on the old `text` schema forever while new - # ones got the honest types — the split schema R3's note warned about, arriving through the - # very function the note was written on. Migrating here means a table is brought forward - # immediately BEFORE anything appends to it, so no caller has to remember anything and no - # tenant is left behind by a script nobody ran. - # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is - # already current, which after the first run is every call. - if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): - try: - migrate_ig_tables(rt, log=lambda *_a: None, only=key) - except Exception: # noqa: BLE001 - # A migration that cannot run must not stop the automation from writing its rows. - # The old schema still reads; a refused write loses the pull we just paid for. - pass - wanted = [] - for raw_field in (fields or []): - field = dict(raw_field) - if flow_tag or lock_fields: - automation = dict(field.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - if lock_fields: - automation["preset"] = True - field["automation"] = automation - wanted.append(field) - created = _iso() - human = username if username and username not in MACHINE_OWNERS else "" - - # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store - # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo - # budget, an idempotent helper that always writes is the same defect as a per-row insert. - have = ut_get(rt, key) if tables is None else tables.get(str(key)) - have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} - missing = [f for f in wanted if f.get("key") not in have_fields] - missing_locks = [f for f in wanted - if lock_fields and f.get("key") in have_fields - and (not isinstance(have_fields[f.get("key")].get("automation"), dict) - or have_fields[f.get("key")]["automation"].get("preset") is not True)] - if (have is not None and not missing and not missing_locks - and not (record_mode and have.get("recordMode") != record_mode) - and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): - return key - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - if len(cur) >= MAX_UT_TABLES: - return cur - cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", - "createdBy": username, "created": created, - "fields": wanted, "rows": {}} - if record_mode: - cur[key]["recordMode"] = record_mode - return cur - have = {f.get("key") for f in (t.get("fields") or [])} - for f in wanted: - if f.get("key") not in have: - t.setdefault("fields", []).append(f) - have.add(f.get("key")) - elif lock_fields: - stored = next((g for g in (t.get("fields") or []) - if g.get("key") == f.get("key")), None) - if stored is not None: - automation = dict(stored.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - automation["preset"] = True - stored["automation"] = automation - # ADOPTION: a machine name is not an owner. It never overwrites a human one. - if human and (t.get("createdBy") or "") in MACHINE_OWNERS: - t["createdBy"] = human - # An automation's table SAYS an automation owns it — the nav badge reads from this. - t["source"] = t.get("source") or "Automation" - if record_mode: - t["recordMode"] = record_mode - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return key - - -def ut_write_rows(rt, key, rows): - """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t["rows"] = rows - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): - """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value - for k, rws in (extra_rows or {}).items(): - tt = cur.get(k) - if tt is not None: - tt["rows"] = rws - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -IG_FIELD_DESCRIPTIONS = { - "alt_text": "Accessibility text attached to the Instagram post.", - # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would - # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED - # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off - # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same - # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a - # description that named only the first would have quietly excluded the second. - "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " - "rather than measured at the time shown.", - "avg_comments_12": "Average comments across the 12 most recent captured posts.", - "avg_engagement": "Average engagement rate reported for the profile.", - "avg_likes_12": "Average likes across the 12 most recent captured posts.", - "avg_plays_12": "Average plays across the 12 most recent captured posts.", - "avg_views_12": "Average views across the 12 most recent captured posts.", - "views": "View count for the post, as Instagram displays it.", - "video_duration": "Length of the video in seconds.", - "comments_disabled": "Checked when the creator has turned comments off for this post.", - "plays": "Times the video started playing, including replays.", - "bio": "Biography shown on the Instagram profile.", - "bio_hashtags": "Hashtags listed in the profile biography.", - "business_category": "Instagram's business category for the account.", - "caption": "Caption published with the Instagram post.", - "category": "Instagram's category for the profile.", - "comment_key": "Unique ID for the captured comment record.", - "commented_at": "Date the comment was posted.", - "comments": "Comment count reported for the post.", - "comments_captured": "Number of distinct comment records linked to this profile.", - "comments_link": "Comment records linked to this record.", - "country_code": "Country code reported for the profile.", - "created_by": "User whose automation first added this lead.", - "enriched_at": "Date the profile was last enriched.", - # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number - # beside it is the only thing that says how much of one. - "location_guess": "Most common city tagged across this profile's captured posts - a guess, " - "not a stated location.", - "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " - "city.", - "external_url": "Website linked from the Instagram biography.", - "external_url_title": "Title Instagram shows for the biography link.", - "fbid": "Facebook ID associated with the Instagram account.", - "first_found": "Date an automation first found this profile.", - "followers": "Latest reported follower count.", - "following": "Latest reported number of accounts followed.", - "found_count": "Number of times discovery automations found this profile.", - "full_name": "Display name shown on the Instagram profile.", - "handle": "Instagram username without the @ symbol.", - "has_channel": "Checked when the profile has an Instagram channel.", - "hashtags": "Hashtags extracted from the post caption.", - "highlights_count": "Total story highlight collections reported for the profile.", - "ig_id": "Instagram's internal ID for the account.", - "influencer_key": "Normalized handle linking this row to its profile.", - "is_business": "Checked when Instagram marks the account as a business.", - "is_joined_recently": "Checked when Instagram marks the account as recently joined.", - "is_private": "Checked when the Instagram profile is private.", - "is_professional": "Checked when Instagram marks the account as professional.", - "last_found": "Date an automation most recently found this profile.", - "likes": "Like count reported for this record.", - "measured_at": "Date engagement metrics on this post were last read.", - "measurements_captured": "Number of measurement rows stored for this post.", - "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", - "partner": "Brand named in the post's paid partnership metadata.", - "partner_id": "Partner ID reported for the Instagram profile.", - "platform": "Social network for this profile.", - "plays": "Video plays, including repeat plays, reported for the post.", - "post_link": "Post record linked to this measurement or comment.", - "post_measurements_captured": "Number of distinct post measurements linked to this profile.", - "post_snapshot_key": "Unique ID for this post measurement.", - "post_snapshots_link": "Engagement measurement rows linked to this record.", - "posted_at": "Date the Instagram post was published.", - "posts_captured": "Number of distinct post records captured for this profile.", - "posts_count": "Total posts reported for the Instagram profile.", - "posts_link": "Post records linked to this profile.", - "profile_name": "Profile name returned by the data provider.", - "profile_reads": "Number of profile snapshots stored for this profile.", - "profile_snapshots_link": "Profile snapshot rows linked to this profile.", - "profile_url": "Direct URL to the Instagram profile.", - "pronouns": "Pronouns shown on the Instagram profile.", - "pulled_at": "Date this snapshot was collected.", - "related_accounts": "Accounts Instagram suggests alongside this profile.", - "replies": "Reply count reported for the comment.", - "shortcode": "Instagram shortcode that uniquely identifies the post.", - "source_payload": "Full source response for this record.", - "snapshot_key": "Unique ID for this profile snapshot.", - "source": "Method or provider used to read this profile.", - "tagged_location": "Location tagged on the post; it is not the creator's residence.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "type": "Instagram media type: image, video, or carousel.", - "url": "Direct URL to the Instagram post.", - "verified": "Checked when Instagram marks the profile as verified.", -} - - -#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the -#: map above is one line of `field_def`: it defaults `description` to -#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column -#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`, -#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram -#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check. -#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting -#: the Instagram sentence is the failure this map exists to prevent. -TT_FIELD_DESCRIPTIONS = { - "platform": "Network this handle is on.", - "handle": "TikTok @name; the unique account handle.", - "full_name": "Display name shown on the TikTok profile.", - "tt_id": "TikTok's own numeric id for the account.", - "profile_url": "Direct URL to the TikTok profile.", - "bio": "Profile biography text.", - "external_url": "Link in the TikTok bio.", - "verified": "Checked when TikTok marks the account as verified.", - "is_private": "Checked when the account is private.", - "followers": "Follower count at the time of the pull.", - "following": "Accounts this profile follows.", - "posts_count": "Videos published by this account.", - "likes_received": "Total likes this account's videos have received.", - "avg_engagement": "Average engagement rate, stored as a percentage.", - "comment_engagement": "Comment engagement rate, stored as a percentage.", - "like_engagement": "Like engagement rate, stored as a percentage.", - "is_business": "Approximate: set when TikTok flags the account as a commerce user.", - "country_code": "Two-letter country code reported for the account.", - "predicted_lang": "Language TikTok predicts for this account.", - "account_created_at": "When the TikTok account itself was created.", - "region": "Region reported for the account.", - "shortcode": "TikTok's numeric id for the video; unique per post.", - "type": "TikTok post type: video or image.", - "url": "Direct URL to the TikTok post.", - "caption": "Post description text.", - "posted_at": "When the creator published the post.", - "likes": "Likes reported for the post.", - "comments": "Comment count reported for the post.", - "views": "Play count TikTok reports for the video.", - "shares": "Times the post was shared.", - "saves": "Times the post was saved to a collection.", - "video_duration": "Video length in seconds.", - "hashtags": "Hashtags used in the post.", - "tagged_location": "Commerce location reported for the post; it is not the creator's home.", - "influencer_key": "Handle of the account this record belongs to.", - "comment_key": "Unique ID for this comment.", - "commented_at": "When the comment was posted.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "replies": "Reply count reported for the comment.", - "snapshot_key": "Unique ID for this profile snapshot.", - "post_snapshot_key": "Unique ID for this post measurement.", - "pulled_at": "When this measurement was read.", - "enriched_at": "When this row was last enriched.", - "source": "Method or provider used to read this record.", - "source_payload": "Full source response for this record.", -} - - -def field_def(text_key, label, ftype="text", **extra): - """One machine-spawned column definition. - - ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, - `profile`), and it stays ONE constructor rather than growing a second for "special" fields. - ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s - W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole - preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). - - ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a - feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so - `core.user_tables._clean_field` — the single validator every field created through the ordinary - door passes — has never judged a single field this engine spawned. The difference was exactly - one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` - for every automation column in the product. - ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an - absent key was already not that, so the bypass was fail-closed and therefore silent. It is - stated now, and `verify_automation` asserts the equality field-by-field, so the next key - `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). - """ - description = " ".join(str(extra.pop( - "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) - out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", - "editRole": "admins"} - if description: - out["description"] = description - out.update(extra) - return out - - -def tt_field_def(text_key, label, ftype="text", **extra): - """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2). - - ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can - never be reached from here. An explicit empty string makes `field_def` omit the key, which is - the same shape an undescribed column already has — the point is that the sentence a TikTok - column carries is one somebody wrote about TikTok, or none at all. - """ - extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, "")) - return field_def(text_key, label, ftype, **extra) - - -# --------------------------------------------------------------------------------------------- -# DEFINITIONS — validation and the bucket's read/write half -# --------------------------------------------------------------------------------------------- - -def _s(v, n=200): - return str(v if v is not None else "")[:n] - - -#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. -#: -#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 -#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config -#: disagree with what the run can possibly do. -#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect -#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve -#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is -#: worse than one that refuses it, because the refusal is the only place the ceiling can be -#: taught. So this REFUSES, and the sentence names the cap and who set it. -#: -#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the -#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the -#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask -#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` -#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a -#: property of the call we happened to make. -#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of -#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE -#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to -#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it -#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. -MAX_POSTS_PER_PULL = 30 -DEFAULT_POSTS_PER_PULL = 10 - -#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately -#: NOT `MAX_POSTS_PER_PULL`. -#: -#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap -#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so -#: raising the capture cap to 30 silently made every one of them average thirty posts under a -#: label that says twelve. Nobody would have looked at those columns again to check. -#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different -#: decisions, and only the first one was made. To widen the average, change this constant AND the -#: four keys and labels together — a column whose meaning changes underneath its own name is -#: worse than one that is merely narrow. -AVG_WINDOW_POSTS = 12 - - -#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a -#: filter cannot ask for a category that can never match a stored row. -POST_TYPES = ("video", "image", "carousel") -#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason -#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video` -#: is the stored key: a client that translated it locally would be a second copy of this -#: vocabulary, free to drift the day a fourth kind appears or a name changes. -#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a -#: label naming only reels would over-promise on a plain video post. -POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"} - - -def clean_post_groups(raw, max_posts): - """`(groups, error)` for `config.postGroups` — "last N reels, last M images". - - ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: - `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. - - ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today - carries no such key, and inventing a filter for it would silently start DROPPING posts those - automations have always captured. Empty list and missing are the same answer. - - ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the - difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them - would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken - one, so the only way to see one is a person typing it, and a filter that silently ignores the - type you asked for is how you end up paying for reels and storing carousels. - """ - if raw in (None, "", [], {}): - return [], None - if not isinstance(raw, list): - return None, "the post groups have to be a list of {type, limit} entries" - out, seen = [], set() - for item in raw: - if not isinstance(item, dict): - return None, "each post group has to be a {type, limit} entry" - t = str(item.get("type") or "").strip().lower() - if t not in POST_TYPES: - return None, (f"'{t or 'blank'}' is not a post type. Use one of " - f"{', '.join(POST_TYPES)}") - if t in seen: - return None, f"the post groups name '{t}' twice. One limit per type" - seen.add(t) - try: - lim = int(item.get("limit")) - except (TypeError, ValueError): - return None, f"how many {t} posts to keep has to be a whole number" - if lim < 1: - return None, f"a {t} group has to keep at least one post" - # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures - # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is - # refused HERE, where the number they typed is still on the screen in front of them. - if lim > max_posts: - return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " - f"group cannot keep {lim}. Raise the post count first") - out.append({"type": t, "limit": lim}) - return out, None - - -def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): - """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. - - ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this - takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old - clamp's default, which this cap now forbids. If an inherited value were refused the same way - a typed one is, every one of those automations would 400 on its next Save **forever**, for a - number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator - that refuses stored data does not protect the user from it, it locks them out of their own - automation. - So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely - INHERITED gets silently brought inside the cap it was already effectively subject to at the - vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default, None - try: - n = int(raw) - except (TypeError, ValueError): - if not submitted: - return default, None - return None, "the post limit must be a whole number" - if not submitted: - return max(1, min(n, MAX_POSTS_PER_PULL)), None - if n < 1: - return None, "keep at least one post per profile, or turn the post capture off" - if n > MAX_POSTS_PER_PULL: - return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts. That is the " - f"vendor's cap, not ours, so {n} would store the same " - f"{MAX_POSTS_PER_PULL} and read as more") - return n, None - - -def clean_config(kind, raw, previous=None): - """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. - - ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail - rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not - edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" - from a panel that never knew about them would silently turn off the dry run, or turn ON a - per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; - present ⇒ take it, including `false`. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - - def flag(name): - return bool(raw[name]) if name in raw else bool(prev.get(name)) - if kind == "plain": - # A plain automation has no machine step, so it only needs the database its records walk. - # - # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on - # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- - # trigger — so requiring a second copy of that fact here would be a mandatory field with - # exactly one legal answer, and a Save that refuses until you retype what you just picked. - # There is no `dryRun`: dry-run means "run the machine step but do not write", and there - # is no machine step to run. - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None - if kind == "scrape_db": - url = _s(raw.get("url"), 2000).strip() - if not url: - return None, "a source URL is required" - try: - guard(url) - except Refused as e: - return None, str(e) - extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" - fmap = {} - for k, v in list((raw.get("fieldMap") or {}).items())[:60]: - tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") - if tk: - fmap[_s(k, 120)] = tk[:60] - if not fmap: - return None, "map at least one source column to a field" - key_field = _s(raw.get("keyField"), 60).strip() - if key_field not in fmap.values(): - return None, "the key field must be one of the mapped fields" - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"url": url, "extract": extract, - "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), - "fieldMap": fmap, "keyField": key_field, - "targetTable": target, - "dryRun": flag("dryRun"), - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None - if kind == "field_instagram": - table = _s(raw.get("targetTable"), 60).strip() - if not table.startswith(UT_PREFIX): - return None, "an Instagram automation runs against a blank database (ut_*)" - fkey = _s(raw.get("fieldKey"), 80).strip() - if not fkey: - return None, "pick the automation column this run writes into" - # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). - # They are read from nothing and written to nothing: a stored definition carrying either - # still SAVES — it simply loses them on the next write — and neither is ever a reason to - # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown - # key would 400 every automation a tenant stored before this wave, forever, on a screen - # that gives them no way to remove it. Dropping a retired key is a migration; refusing it - # is an outage. - # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary - # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired - # USER concept — renaming them would be a second, unrelated change wearing this one's - # justification. - # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel - # that does not edit the post count must not reset it to the default on Save. And an - # INHERITED value is clamped rather than refused; see `clean_max_posts`. - sent = "maxPosts" in raw - max_posts, perr = clean_max_posts( - raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) - if perr: - return None, perr - return {"targetTable": table, "fieldKey": fkey, - "urlField": _s(raw.get("urlField"), 80).strip(), - # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked - # for: the Profiles row carries post IDENTITY for free but NO engagement - # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. - "postMetrics": flag("postMetrics"), - # The full Comments dataset can bill many rows per post. It is an explicit, - # independent opt-in and never follows the per-post switch automatically. - "commentMetrics": flag("commentMetrics"), - "dryRun": flag("dryRun"), - "maxPosts": max_posts}, None - # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor - # the same question of two different corpora: a record ceiling, a predicate list, a join word - # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the - # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write - # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists - # to prevent, and it would do it silently. - if kind in DISCOVERY_KINDS: - # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now - # come from `discovery_facts` rather than being spelled out here. They were correct; they - # were also the FIFTH copy of "which table does this kind write to", and the other four - # were the ones wave 29 forgot to update. - _, _disc_table, _disc_label, _ = discovery_facts(kind) - limit = int(raw.get("recordsLimit") or 0) - if limit < 1: - return None, ("say how many profiles to fetch. An UNBOUNDED discovery query is the " - "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") - if limit > BD_MAX_RECORDS: - return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what - # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading - # the operator afterwards is how the OR hole survived: the guard was handed the branches - # and never told they were a union. - join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" - # ⭐ WAVE 32 · T46 (D-167) — AND THE KIND GOES WITH THEM. `clean_config` already knows which - # corpus this automation searches; passing it is what stops a `discover_tiktok` being built - # on the 16 Instagram fields TikTok's dataset does not carry — a search that does not error, - # returns nothing, and reads as "no such creators exist" after the money is spent. - preds, perr = clean_predicates(raw.get("predicates"), join, kind) - if perr: - return None, perr - target = _s(raw.get("targetTable"), 60).strip() or _disc_table - if not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) - if serr: - return None, serr - return {"recordsLimit": limit, "predicates": preds, - "operator": join, - "targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, - # C6/R5: WHERE the conditions above came from, when they were derived. Stored so - # the surface can say "these were filled in from the 'Florists' view, over 12 - # records" instead of presenting them as if somebody typed them. - "seed": seed, - "dryRun": flag("dryRun")}, None - return None, f"unknown automation kind {kind!r}" - - -def clean_seed(raw): - """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter - somebody typed by hand has no seed, and that is the ordinary case. - - ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that - the derived conditions are "written into `config.predicates` as ordinary conditions — it is a - filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, - like every other search. Keeping it means the surface can say where those rows came from, and - a user editing them freely is exactly what is supposed to happen. - """ - if raw in (None, "", {}): - return {}, None - if not isinstance(raw, dict): - return None, "the seed must be an object" - source = _s(raw.get("source"), 12).strip().lower() - if source not in SEED_SOURCES: - return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) - table = _s(raw.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, f"a seed reads a blank database (ut_*). {table!r} is not one" - basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} - derived, _err = clean_predicates(raw.get("derived") or [], "and") - return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), - # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what - # was suggested; the conditions that matter are already in `predicates` and were - # validated there. Refusing a Save because a stored provenance note aged badly would - # block the user from editing the very filter it describes. - "derived": derived or [], - "basis": {"rows": max(0, int(basis.get("rows") or 0)), - "fields": [f for f in (basis.get("fields") or []) - if isinstance(f, dict)][:SEED_MAX_PREDICATES], - "related": basis.get("related") - if isinstance(basis.get("related"), dict) else {}, - "note": _s(basis.get("note"), 200)}}, None - - -def clean_schedule(raw, previous=None): - """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` - measures from — without it, enabling a daily job fires it immediately).""" - raw = raw if isinstance(raw, dict) else {} - cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" - parse_cron(cron) # raises -> the route answers 400 - enabled = bool(raw.get("enabled")) - prev = previous or {} - out = {"cron": cron, "enabled": enabled} - if enabled: - out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() - return out - - -# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── -# The owner's words: "we can make this automation a loop, so the ending should always be defined, -# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, -# or the user have to click a button to reset, or after a certain amount of time it can -# automatically reset to first cycle." -# -# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition -# that predates this field must not start moving records on its own the day the code ships. A -# loop is a thing somebody turns on. -def clean_flow(raw, previous=None, notes=None, rt=None): - """Validate the builder's ordered action list. `(flow, error)`. - - ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions( - raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) - if err: - return None, err - return {"actions": actions}, None - - -#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the -#: definition carries none. -#: -#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at -#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the -#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly -#: created automation. Two numbers for one fact, in the panel, before anybody had typed -#: anything. **Read off the screenshot; every assertion in the battery was green.** -#: -#: This is the same species as the default-versus-guard split fixed the same day: a value -#: decided in one file and a value decided in another, describing the same thing. Aligning the -#: constants closes the only window in which they can disagree — the seed — because every later -#: state comes from the stored config. -DISCOVER_SEED_RECORDS = 25 - - -def discovery_seed_spec(kind): - """-> (default table, enrich action kind) for a discovery KIND. - - ⭐⭐ WAVE 30, OWNER REPORT 2026-08-12, verbatim: *"'When a Tiktok profile fits a criteria' - should ALWAYS have a 'Create record' EXACTLY like the Instagram one … THE ONLY DIFFERENCE IS - THE COLUMNS AND DATA SCHEMA"*. They were right, and the seeding path was the one family of - sites this wave widened everywhere ELSE: `clean_definition` planted step 1 and step 2 only when - the trigger was `ig_profile_match`, so a TikTok search stored `flow.actions = []` and had - nowhere to put what it found — MEASURED live against `auto_1` (Instagram: create_record + - enrich) versus a fresh TikTok discovery (empty). - - ⭐ THIS IS A LOOKUP, NOT A SECOND SEEDER, and that is the whole point of the ruling. One - builder plants both platforms' steps; the only things that vary are the table the record lands - in and which enrich action reads it — literally "the columns and data schema". A forked - `_ensure_tt_action` would start identical and drift, which is the failure `ENRICH_KINDS` and - `DISCOVERY_KINDS` were introduced to prevent one screen up. - - ⚠ Resolved in a FUNCTION rather than a module-level dict because `DISCOVER_TABLE` is defined - far below this line; a dict here would raise at import. - """ - if kind == "discover_tiktok": - return TT_PROFILE_TABLE, "enrich_tiktok" - return DISCOVER_TABLE, "enrich_instagram" - - -def _ensure_ig_action(flow_raw, table, enrich_kind="enrich_instagram"): - """Create record is PERMANENT step 1 on a DISCOVERY trigger — it cannot be deleted or moved. - - ⚠ THE NAME IS INSTAGRAM'S AND THE BEHAVIOUR IS BOTH PLATFORMS' (wave 30). Renaming it would - churn six gate references for no behaviour change; `enrich_kind` is what makes it general, and - it defaults to Instagram's so every pre-existing caller keeps its exact meaning. - - ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, - on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a - comment warning that re-seeding on every clean would be "a control that will not take no for - an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be - deleted, because the nature of that automation is that it needs to first create a record in a - database somewhere from the list of profiles to fetch."* - - Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER - produces rows has nowhere to put them until something writes them, so an Instagram search - with no Create record is not a customised automation — it is a search whose results are - discarded. That is not a preference to respect. - - ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and - the values it maps are theirs. An existing `create_record` further down is MOVED to the front - rather than duplicated — re-seeding a second one would quietly double every run's writes. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if actions and actions[0].get("kind") == "create_record": - return _pin_unique(_ensure_enrich_step( - flow_raw if isinstance(flow_raw, dict) else flow, enrich_kind)) - at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) - if at > 0: - actions.insert(0, actions.pop(at)) - else: - # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner - # and no card: the wave-23 header records four kinds that shipped with illegal seeds and - # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. - actions.insert(0, { - "id": "act_1", "kind": "create_record", "enabled": True, "when": None, - # `config.label` follows the `review` action's precedent in `_clean_action_config` — - # an action naming itself, inside the untyped config bag, so no shared TS type changes. - "config": {"table": str(table or DISCOVER_TABLE), - "label": "Save the profile", - # C5: the picture and the real write finally agree — see `_pin_unique`. - "uniqueOn": "handle", - "values": {"handle": "{{handle}}"}}, - }) - flow["actions"] = actions - return _pin_unique(_ensure_enrich_step(flow, enrich_kind)) - - -def _ensure_enrich_step(flow_raw, enrich_kind="enrich_instagram"): - """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON A DISCOVERY SEARCH. - - ⚠ WAVE 30: `enrich_kind` selects the network. Instagram's is the default so every pre-existing - caller means exactly what it meant before; TikTok passes `enrich_tiktok` and gets the identical - step shape, which is the owner's *"only the columns and data schema differ"*. - - Owner: *"make it default that this enrichment action is Step 2 always, and under Config of - Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for - the same reason: a search that finds profiles and never reads them has done half a job. The - difference is the control — step 1 is permanent because a flow without it discards its - results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and - disabling would be two ways to say one thing, and only one of them survives a re-save. - - ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains - this step by upgrading does not quietly start spending; switching the Source to the paid - provider is an explicit choice a person makes in front of the sentence that names the cost. - ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — - a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow - re-buying the same profile nightly. Off-by-default would make the expensive behaviour the - accident. - - ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an - If/then branch has one; seeding a second at the top level would enrich twice and bill twice, - which is the duplication `apply_actions` already paid for once with the pinned step 1. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if any(a.get("kind") == enrich_kind for a in walk_actions(actions)): - return flow_raw if isinstance(flow_raw, dict) else flow - seeded = list(actions) - # Index 1 — after the pinned Create record, because there is nothing to enrich until the - # profiles have been written as records. `insert` past the end is a plain append, so a flow - # with only step 1 lands this at the end, which IS step 2. - seeded.insert(1, { - "id": "act_enrich", "kind": enrich_kind, "enabled": True, "when": None, - "config": {"postMetrics": False, "commentMetrics": False, - "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, - "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", - "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, - "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, - }) - return {**flow, "actions": seeded} - - -#: The key the discovery writer really upserts on. Both discovery runners key candidates on -#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's -#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is -#: supplied by the runner, never typed by a person. -#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that -#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is -#: the pair above and the TENANT is the unit, while `created_by` survives as an informational -#: "Found by" stamp that no run may branch on. -#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note -#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate -#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a -#: bug with a rationale attached. -IG_PINNED_UNIQUE = "handle" - - -def _pin_unique(flow): - """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. - - ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — - the builder posts the whole action list on every save — so a client that omitted the key would - silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's - own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming - "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this - module refuses everywhere else. - - ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason - this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action - does not write, so the stamp is applied ONLY when `handle` is among the action's own values. - A user who remaps the card to write different columns keeps their edit and still saves; the - alternative — stamping unconditionally — is a product-seeded default that its own validator - would 400, which is precisely the post-W24 hotfix this rule exists because of. - - ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes - `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED - definition's own dict, so writing into it would edit the live store object in memory, before - (and regardless of) any commit. Every touched level is copied instead. - """ - if not isinstance(flow, dict): - return flow - acts = flow.get("actions") - if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): - return flow - first = acts[0] - if first.get("kind") != "create_record": - return flow - cfg = first.get("config") - if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): - return flow - if str(cfg.get("uniqueOn") or "").strip(): - return flow - return {**flow, - "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} - - -#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so -#: the seeder and the un-seeder cannot disagree about what "seeded" means. -#: ⚠ WAVE 30 — `enrich_tiktok` shares `act_enrich`. The map is keyed by KIND, and only one enrich -#: step is ever seeded per flow (the network follows the trigger), so the id cannot collide. -_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich", - "enrich_tiktok": "act_enrich"} - - -def _is_untouched_ig_seed(action, table, enrich_kind="enrich_instagram"): - """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) - - ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and - that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build - the same dicts one screen above, so a step gaining a config key next wave gains it here too. - A remembered key list would quietly start calling every seeded action "edited". - - ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The - stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the - kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW - dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", - and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only - version where the two sides are the same kind of object. - - ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the - TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about - a step they still want, and an off seed is still a seed. - """ - if not isinstance(action, dict): - return False - kind = str(action.get("kind") or "") - if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): - return False - seeded, _seed_err = clean_actions( - (_ensure_ig_action({"actions": []}, table, enrich_kind).get("actions") or [])) - fresh = {a.get("kind"): a for a in (seeded or [])} - seed = fresh.get(kind) - if not seed: - return False - cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) - if kind in ENRICH_KINDS: - cfg.pop("enabled", None) - seed_cfg.pop("enabled", None) - return cfg == seed_cfg and (action.get("when") or None) is None - - -def _drop_ig_seeds(flow_raw, table, enrich_kind="enrich_instagram"): - """Strip the trigger's own seeded steps, keeping any the person has since made theirs. - - ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT - THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it - does not delete the seeded actions … a create_record the person has since re-pointed at their - own table with their own values is THEIR action now"*. That reasoning is still correct and is - exactly what this function preserves — but it was applied to EVERY seeded action, including - the ones nobody had ever opened, and the result is what the owner reported: switch an - Instagram search to Manual and you are left holding a "Save the profile" step writing - `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not - somebody's work being protected; it is the machine's own leftover, pointed at a table the - automation has nothing to do with any more. - - ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as - `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a - branch that destroys something, so the conservative half is the load-bearing half. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - kept = [a for a in actions if not _is_untouched_ig_seed(a, table, enrich_kind)] - if len(kept) == len(actions): - return flow_raw - return {**flow, "actions": kept} - - -def ig_action_pinned(defn, index): - """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` - flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" - # ⭐ WAVE 30 — BOTH discovery kinds. It read `== "discover_instagram"`, so TikTok's step 1 (once - # seeded) would have rendered as an ordinary draggable, deletable card: the same rule the owner - # asked for "EXACTLY", applied to only one network. - return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 - - -def clean_definition(raw, previous=None, username="", notes=None, rt=None): - """Whole-definition validation. Returns `(defn, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous or {} - # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing - # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second - # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys - # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload - # invalid in BOTH its trigger and its config now answers with the trigger's sentence. - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr - # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this - # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a - # dead end the moment the wizard was deleted and the client's create body became {name}. - kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND - drop_ig_seeds = False - if (trigger or {}).get("key") == "ig_profile_match": - # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather - # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and - # whose kind says something else is not a preference to respect, it is two halves of one - # answer disagreeing, and the trigger is the half the person actually picked. - kind = "discover_instagram" - elif (trigger or {}).get("key") == "tiktok_profile_match": - # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how - # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram - # arm is — a definition whose trigger says TikTok discovery and whose kind says something - # else is two halves of one answer disagreeing. - kind = "discover_tiktok" - elif (kind == "discover_tiktok" - and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match" - and (trigger or {}).get("key") != "tiktok_profile_match"): - # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm - # below): with nothing to flip the kind BACK, switching the trigger to Manual left - # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus - # search on a flow the person had just made manual.** Shipped here in the same change as - # law 1 rather than discovered the same way twice. - # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the - # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because - # `patch()` builds its raw as `dict(prev)` plus the caller's keys. - kind = DEFAULT_KIND - # ⭐ WAVE 30 — un-seed on the way out, exactly as the Instagram arm below does. This line - # was ABSENT and harmless for as long as TikTok had no seeds to leave behind; the moment - # the seeder above learned TikTok, its absence became wave-27 item 15's defect on the other - # network — a flow switched to Manual still carrying a "Save the profile" step nobody - # planted deliberately. Found by widening the seeder and asking what else assumed only - # Instagram could have seeds. - drop_ig_seeds = True - elif (kind == "discover_instagram" - and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" - and (trigger or {}).get("key") != "ig_profile_match"): - # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. - # - # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram - # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch - # the trigger to Manual and the automation stayed `discover_instagram`, which means - # - # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow - # the person had just made MANUAL fired a PAID Bright Data corpus search; - # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed - # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the - # instagram trigger, the 'always first' create record just stuck there"*. The server - # would have accepted the delete; the CLIENT hid the control, because both halves ask - # the kind and the kind was lying. - # - # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in - # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as - # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY - # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. - # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` - # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest - # question: did this automation stop being an Instagram search? - # - # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram - # trigger — a `discover_instagram` created with an explicit kind and some other trigger is - # somebody's deliberate state, not a mistake to correct. And only FROM - # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose - # triggers are their own business, and clobbering them would retire them by accident. - # - # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this - # replaces argued the other way, so here is why it was half right. It read: *"it does not - # delete the seeded actions … a create_record the person has since re-pointed at their own - # table with their own values is THEIR action now, and quietly destroying it is the silent - # data loss this module refuses everywhere else."* Every word of that is still true and is - # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the - # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to - # Manual still carrying a "Save the profile" step writing `{{handle}}` into - # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces - # handles, unpinned but still there, still runnable, and still pointed at a table that has - # nothing to do with this automation. - # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. - kind = DEFAULT_KIND - drop_ig_seeds = True - if kind not in KINDS: - return None, f"unknown automation kind {kind!r}" - # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. - # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` - # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. - # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way - # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two - # live automations save through this function every time their owner edits them. The gate's - # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved - # from `create` into `clean_definition` (the tempting simplification), this helper would go - # red across a dozen sections, which is the alarm that change deserves."* It did, and the - # alarm worked. - # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every - # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody - # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an - # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored - # ones are rewritten under their owner — and that is a ruling, not a refactor. - name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) - if not name: - return None, "name the automation" - cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - if kind in DISCOVERY_KINDS and prev.get("kind") != kind: - # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS - # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a - # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an - # UNBOUNDED discovery query is the one shape the vendor refuses outright - # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the - # owner saw as "the Trigger won't load". - # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends - # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` - # at that moment, so it falls through to `return { targetTable }`). IG worked only because - # this seed caught it. So the defect was never "TikTok is missing something Instagram has" - # — it was one hard-coded string in the single line that rescues both. - # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old - # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a - # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). - # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very - # first Save after picking this trigger 400s — against the stored-inert-with- - # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). - # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard - # exists because an unbounded query is the one shape the vendor refuses outright, and - # coercing a blank into a number would be exactly the silent widening it protects against. - # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until - # a person presses Run now or arms the schedule. - cfg_raw = dict(cfg_raw or {}) - if not _ig_int(cfg_raw.get("recordsLimit")): - cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS - config, err = clean_config(kind, cfg_raw, prev.get("config")) - if err: - return None, err - if trigger: - # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own - # configuration IS the config, and `clean_trigger` could not see it. - trigger["configured"] = _trigger_configured(trigger, config) - try: - schedule = clean_schedule( - raw.get("schedule") if isinstance(raw.get("schedule"), dict) - else prev.get("schedule"), prev.get("schedule")) - except ValueError as e: - return None, str(e) - # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) - flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") - # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only - # call was what made the action deletable; running it on every save is what makes step 1 - # permanent, and it is deliberate rather than a widened condition nobody noticed. - # ⭐⭐ WAVE 30 (owner, 2026-08-12) — BOTH DISCOVERY TRIGGERS SEED, not just Instagram's. This - # single `==` was the whole of *"why is it not copied EXACTLY"*: a TikTok search stored an - # EMPTY flow, so the product's own rule — a search that finds profiles must have somewhere to - # put them — held for one network and not the other. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - _seed_table, _seed_enrich = discovery_seed_spec(kind) - flow_raw = _ensure_ig_action( - flow_raw, config.get("targetTable") or _seed_table, _seed_enrich) - elif drop_ig_seeds: - # ITEM 15: the trigger just STOPPED being a discovery one. Un-seed what the trigger - # planted and nobody has since made theirs — measured against the table the seeds were - # planted for, which is the PREVIOUS config's, not the one being saved. - # ⚠ And against the PREVIOUS kind's enrich action, for the same reason: the seeds to strip - # are TikTok's if that is what was planted, and asking "is this Instagram's seed?" of a - # TikTok flow answers no and silently leaves the leftover behind. - _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) - flow_raw = _drop_ig_seeds( - flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) - flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) - if ferr: - return None, ferr - # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that - # its `config.table` is authoritative has to reach it, or the two fields that name one database - # would disagree on exactly the network that just gained the card. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and - # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since - # wave 24: the pinned step 1 says where profiles are written, and the config says where - # the automation's records live — and for a discovery flow those are the same database by - # construction. When they disagreed, every surface picked a different winner (the canvas - # Write node read the config, the actual write read the action), which is a bug you can - # only see by comparing two panels. - # - # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the - # action's own configuration, so the table named there is the one they were looking at. - # - # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: - # a stored definition carrying two different values is reconciled on its next clean, - # silently and exactly once, because nothing writes the losing value back. A refusal here - # would 400 every Save of a definition that was legal yesterday. - # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the - # TIE-BREAK rather than the whole rule. The owner's report: change the database on the - # TRIGGER, save, and the picker springs back to the old one with no message. The cause is - # the branch below read unconditionally — the pinned action still named yesterday's table, - # so it overwrote a change the person had just made in front of it, silently, every time. - # - # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled - # "when the two disagree, whose value is real?" — the action's, because R2b puts the - # preset list on the action's own panel, so that is the one they were looking at. That - # answers a definition at REST. A save is different: it carries an INTENT, and the honest - # question is which side this particular save changed. So: - # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to - # follow it (leaving the action behind would just re-revert on the next save); - # · only the action moved, or neither did and they were already inconsistent -> R2's - # migration, unchanged, reconciling a stored definition exactly once. - # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is - # also the only reading that cannot lose a value: the action's table is the one with - # the field mapping attached to it. - first = (flow.get("actions") or [{}])[0] - pinned = str((first.get("config") or {}).get("table") or "") \ - if first.get("kind") == "create_record" else "" - prev_target = str((prev.get("config") or {}).get("targetTable") or "") - prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] - ).get("config") or {}).get("table") or "") - target_moved = bool(prev_target) and config.get("targetTable") != prev_target - action_moved = bool(prev_pinned) and pinned != prev_pinned - if pinned and target_moved and not action_moved: - # The person changed the trigger's database. Carry it INTO the pinned step so the two - # agree, instead of throwing their edit away to keep a stale copy consistent. - cfg1 = dict(first.get("config") or {}) - cfg1["table"] = str(config.get("targetTable") or "") - flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] - elif pinned and pinned != config.get("targetTable"): - config["targetTable"] = pinned - # The stored label described a DIFFERENT database, so keeping it would caption the new - # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` - # never relabels a table that already exists — so an adopted hand-made database keeps - # the name its owner gave it. - config["targetLabel"] = "" - return { - "id": _s(prev.get("id") or raw.get("id"), 40), - "name": name, "kind": kind, "config": config, "schedule": schedule, - "trigger": trigger, - # The Builder's ordered actions. An absent flow is simply an automation with no actions. - "flow": flow, - "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, - "runs": list(prev.get("runs") or [])[:MAX_RUNS], - # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 - # minutes to build, so a run persists its id here and the NEXT run collects it. It is - # carried through `patch` untouched because a user pressing Save must not silently orphan - # a set the vendor is already building (and already counting against the funds gate). - "state": dict(prev.get("state") or {}), - "created": prev.get("created") or _iso(), - "createdBy": prev.get("createdBy") or username, - # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. - # - # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. - # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded - # system agent undeletable — and this function is a WHITELIST that did not carry the key, so - # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The - # seed would then mint it again on the next list call: a delete that appears to work and - # undoes itself. Found by asserting the round trip rather than by reading the code. - # - # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST - # `{"system": "anything"}` and mint themselves an automation nobody can delete — a - # privilege escalation through a field nobody validates. The flag is granted by the seeder - # and inherited from the stored row, never asserted by a payload. - # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already - # describes as "sticky across a PATCH". One idea, one behaviour, two objects. - **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), - }, None - - -def set_state(rt, auto_id, patch_state): - """Merge into ONE automation's engine state. A key set to None is removed. - - Its own tiny store write rather than a field on `_commit_run`, because the handoff must - survive a run that ends in `error` — a snapshot the vendor is building does not stop existing - because the run that started it failed afterwards. - """ - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - st = dict(d.get("state") or {}) - for k, v in (patch_state or {}).items(): - st.pop(k, None) if v is None else st.__setitem__(k, v) - d["state"] = st - return cur - - # ⚠ W31-T35 — THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a - # trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here - # would re-read the whole bucket one row later than before and undo D-134 entirely. Safe - # because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write - # into the memo in the same statement. Read `_store_update`'s note before adding a second. - _store_update(rt, _up, flush="sync", keeps_defs=True) - - -def _without_retired_board(definition): - """Return a definition with retired Board-only state removed, without mutating the store. - - This protects scheduled/manual execution before the next UI list request persists the - migration. It removes only the former Board configuration, actions, and audit trail; normal - actions and all user data remain intact. - """ - if not isinstance(definition, dict): - return definition, False - out, changed = dict(definition), False - cfg = out.get("config") - if isinstance(cfg, dict) and "lanes" in cfg: - cfg = dict(cfg) - cfg.pop("lanes", None) - out["config"] = cfg - changed = True - - def _actions(actions): - nonlocal changed - clean = [] - for action in actions if isinstance(actions, list) else []: - if not isinstance(action, dict): - clean.append(action) - continue - if action.get("kind") == "review": - changed = True - continue - item = dict(action) - if item.get("kind") == "group" and isinstance(item.get("config"), dict): - group_cfg = dict(item["config"]) - branches = group_cfg.get("branches") - if isinstance(branches, list): - kept = [] - for branch in branches: - if not isinstance(branch, dict): - changed = True - continue - next_actions = _actions(branch.get("actions")) - if not next_actions: - changed = True - continue - kept.append({**branch, "actions": next_actions}) - if not kept: - changed = True - continue - if kept != branches: - changed = True - group_cfg["branches"] = kept - item["config"] = group_cfg - elif isinstance(group_cfg.get("actions"), list): - nested = _actions(group_cfg["actions"]) - if not nested: - changed = True - continue - if nested != group_cfg["actions"]: - group_cfg["actions"] = nested - item["config"] = group_cfg - clean.append(item) - return clean - - flow = out.get("flow") - if isinstance(flow, dict): - next_flow = dict(flow) - actions = _actions(flow.get("actions")) - if actions != flow.get("actions"): - next_flow["actions"] = actions - if "ending" in next_flow: - next_flow.pop("ending", None) - changed = True - out["flow"] = next_flow - if "reviews" in out: - out.pop("reviews", None) - changed = True - return out, changed - - -def retire_automation_board_state(rt, tables=None): - """Persist the idempotent Board retirement, then remove its generated table fields. - - ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches - the stage scan and nothing else. - """ - fields = retire_automation_stage_fields(rt, tables=tables) - try: - stored = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {"definitions": 0, **fields} - plan = {str(aid): cleaned for aid, raw in stored.items() - for cleaned, changed in [_without_retired_board(raw)] if changed} - if not plan: - return {"definitions": 0, **fields} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for aid, cleaned in plan.items(): - if aid in cur: - cur[aid] = cleaned - return cur - - _store_update(rt, _up, flush="sync") - return {"definitions": len(plan), **fields} - - -#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe. -#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which -#: exactly one caller uses (`grid_hook`). -_DEFS_MEMO = {} -#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a -#: cache — an import of 20,000 rows arrives in far less than this, and a stale trigger for two -#: seconds is bounded and recoverable where a stale one for a minute is a mystery. -_DEFS_TTL = 2.0 + touched.append(str(aid)) + return cur + + if key: + _store_update(rt, _up, flush="sync") + return touched + + +def ut_get(rt, key): + return ut_all(rt).get(str(key)) + + +def retire_automation_stage_fields(rt, tables=None): + """Delete obsolete Board-only fields and their hidden cells, never user fields. + + The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` + metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old + generated timestamp/cycle cells are cleared too, including rows whose field definition was + removed by an interrupted earlier migration. Re-running this migration is a no-op. + + ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is + O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to + deep-copy (see this module's header), and `GET /automations` was paying for THREE independent + copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the + WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent + snapshot can never be the thing that gets written back. + """ + tables = ut_all(rt) if tables is None else tables + stage_keys = set() + for table in tables.values(): + if not isinstance(table, dict): + continue + for field in table.get("fields") or []: + auto = field.get("automation") if isinstance(field, dict) else None + if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): + stage_keys.add(str(field.get("key") or "")) + for row in (table.get("rows") or {}).values(): + for key in (row or {}): + text = str(key) + if text.startswith("stage_auto_"): + stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) + stage_keys.discard("") + if not stage_keys: + return {"tables": 0, "fields": 0, "cells": 0} + + changed = {"tables": set(), "fields": 0, "cells": 0} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for table_key, table in cur.items(): + if not isinstance(table, dict): + continue + kept, removed = [], set() + for field in table.get("fields") or []: + auto = field.get("automation") if isinstance(field, dict) else None + key = str(field.get("key") or "") if isinstance(field, dict) else "" + if key in stage_keys and isinstance(auto, dict) and \ + (auto.get("stageField") or auto.get("cyclesField")): + removed.add(key) + changed["fields"] += 1 + continue + kept.append(field) + if removed: + table["fields"] = kept + changed["tables"].add(str(table_key)) + for row in (table.get("rows") or {}).values(): + if not isinstance(row, dict): + continue + for stage_key in stage_keys: + for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): + if key in row: + row.pop(key, None) + changed["cells"] += 1 + changed["tables"].add(str(table_key)) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return {"tables": len(changed["tables"]), "fields": changed["fields"], + "cells": changed["cells"]} + + +def bind_unbound_fields(rt): + """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` + are bound to the definition that already writes them — matched on the (targetTable, + fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no + definition references is DISABLED with the reason on it rather than deleted or guessed: + the column was already dead (nothing runs a column no flow names), and now it says so. + Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" + by_binding = {} + for aid, d in all_definitions(rt).items(): + cfg = d.get("config") or {} + if cfg.get("fieldKey") and cfg.get("targetTable"): + by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) + plan = {} + for tk, t in ut_all(rt).items(): + for f in (t.get("fields") or []): + a = f.get("automation") + # An already-DISABLED bag is a decision this migration made on a previous pass — + # re-planning it every call would turn the once-per-process sweep into a write + # per read. + if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ + and not a.get("disabled"): + plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) + if not plan: + return 0, 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for (tk, fk), aid in plan.items(): + for f in ((cur.get(tk) or {}).get("fields") or []): + if f.get("key") == fk and isinstance(f.get("automation"), dict): + if aid: + f["automation"]["flowId"] = aid + else: + f["automation"]["disabled"] = True + f["automation"]["statusNote"] = ( + "not bound to any flow. Create an automation for this column " + "or delete it") + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + bound = sum(1 for v in plan.values() if v) + return bound, len(plan) - bound + + +def ut_key_for(label, key=None): + """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which + must not create anything) resolves the same key by construction rather than by copying the + derivation and drifting from it.""" + k = str(key or (UT_PREFIX + _ut_slug(label))) + return k if k.startswith(UT_PREFIX) else UT_PREFIX + k + + +#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same +#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging +#: `core` into the API's boot path — and the two are held in step by a gate check. +MACHINE_OWNERS = ("automation", "scheduler") + + +def _preset_description_due(stored, wanted): + """The shipped sentence a stored PRESET column is still missing, or None to leave it alone. + + ⭐⭐ WAVE 41 · T22 (R19: *a user's description edit wins forever*) — `IG_FIELD_DESCRIPTIONS` + and `TT_FIELD_DESCRIPTIONS` reach a BRAND-NEW column through `field_def`, which is why an + untouched field has always shown the right sentence. They reach an EXISTING one only through + `_reconcile_ig_graph_fields` — and that pass is Instagram's. **TikTok has no reconciler at + all**, so a `ut_tt_*` column that predates its sentence (or one whose description was cleared) + had nothing anywhere that could put the default back, and R19's *Reset to default* would have + had nothing to restore FROM. `ut_ensure(lock_fields=True)` is the one door BOTH platforms' + preset sets pass through, so the repair belongs there; on Instagram it is a no-op after the + reconciler has already agreed with it. + + ⛔ THE CUSTODY STAMP IS THE WHOLE POINT, and it is `user_tables.user_edited` — the SAME + predicate `_reconcile_ig_graph_fields` reads, never a second mechanism invented beside it. A + human who has taken a column over keeps their sentence forever, and CLEARING the stamp is what + makes the shipped one come back. That is the reset, expressed as the absence of custody rather + than as a second flag somebody has to remember to write. + ⚠ THE STAMP IS FIELD-WIDE, not description-only (`core.user_tables.USER_EDITED_KEY`, written + once in `patch_field`). This function therefore cannot tell a description edit from a label + edit, and deliberately does not try: it reads the custody mark that exists. + ⚠ NORMALISED AND CAPPED exactly as `core.user_tables._clean_field` would, so the value written + here is the value that validator would keep — a preset whose stored sentence disagreed with + its own validator would otherwise be rewritten on every single run. + """ + want = " ".join(str((wanted or {}).get("description") or "").split())[:300] + if not want or _ut().user_edited(stored): + return None + return None if str((stored or {}).get("description") or "") == want else want + + +def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", + record_mode="", lock_fields=False, tables=None): + """Create the table if it is missing; return its key. Idempotent — a re-run of an automation + that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) + and an existing table with that key is adopted, not duplicated. + + ⭐⭐ WAVE 31 · T30 — `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is + the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every + call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass + paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still + takes forever to change the Config for automation as well. It just say Saving..."* — measured + **7,912 ms** for one save on `4258a93`. + + ⚠ THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope — the same argument + `retire_automation_stage_fields` (this module, same parameter name, same contract) already + makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`, + whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore + never be the thing that gets written back**, and the worst a stale one can do is spend a commit + that would have been skipped — never write a wrong value. Callers that ensure two tables + under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`). + + Fields are MERGED, never replaced: a user who added a column to an automation's table keeps + it, and a new source column joins on the next run. + + `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped + `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their + provenance. Fields already on the table keep whatever tag they have (first flow wins; + shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not + ownership). + + ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever + or WHATEVER ran the automation, so the same table belonged to a person if its first run was + manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` + admits only the creator or an admin, so **whether you could open your own Instagram + snapshots depended on a race you never saw**. The owner is now the automation's creator, and + a table already stamped with a machine name is ADOPTED the next time a run knows a human + one. Adoption is a repair, not a widening: the automation's creator is the person who asked + for the table in the first place. + """ + key = ut_key_for(label, key) + # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. + # + # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it + # would leave every table already in production on the old `text` schema forever while new + # ones got the honest types — the split schema R3's note warned about, arriving through the + # very function the note was written on. Migrating here means a table is brought forward + # immediately BEFORE anything appends to it, so no caller has to remember anything and no + # tenant is left behind by a script nobody ran. + # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is + # already current, which after the first run is every call. + if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): + try: + migrate_ig_tables(rt, log=lambda *_a: None, only=key) + except Exception: # noqa: BLE001 + # A migration that cannot run must not stop the automation from writing its rows. + # The old schema still reads; a refused write loses the pull we just paid for. + pass + wanted = [] + for raw_field in (fields or []): + field = dict(raw_field) + if flow_tag or lock_fields: + automation = dict(field.get("automation") or {}) + if flow_tag: + automation.setdefault("flowId", str(flow_tag)) + if lock_fields: + automation["preset"] = True + field["automation"] = automation + wanted.append(field) + created = _iso() + human = username if username and username not in MACHINE_OWNERS else "" + + # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store + # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo + # budget, an idempotent helper that always writes is the same defect as a per-row insert. + have = ut_get(rt, key) if tables is None else tables.get(str(key)) + have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} + missing = [f for f in wanted if f.get("key") not in have_fields] + missing_locks = [f for f in wanted + if lock_fields and f.get("key") in have_fields + and (not isinstance(have_fields[f.get("key")].get("automation"), dict) + or have_fields[f.get("key")]["automation"].get("preset") is not True)] + # ⭐ WAVE 41 · T22 — AND THE SKIP TEST HAS TO KNOW ABOUT THE DESCRIPTION REPAIR, or the repair + # in `_up` below can never run: every table that already exists returns HERE, three lines + # before it. A term added to `_up` without a term added here is a change that reads correctly + # and does nothing on precisely the tables it was written for. + # ⚠ It converges: once `_up` has written the sentence, `_preset_description_due` answers None + # for that field and this skip fires again, so the owner's save path pays no extra commit. + stale_descriptions = [f for f in wanted + if lock_fields and f.get("key") in have_fields + and _preset_description_due(have_fields[f.get("key")], f) is not None] + if (have is not None and not missing and not missing_locks and not stale_descriptions + and not (record_mode and have.get("recordMode") != record_mode) + and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): + return key + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is None: + if len(cur) >= MAX_UT_TABLES: + return cur + cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", + "createdBy": username, "created": created, + "fields": wanted, "rows": {}} + if record_mode: + cur[key]["recordMode"] = record_mode + return cur + have = {f.get("key") for f in (t.get("fields") or [])} + for f in wanted: + if f.get("key") not in have: + t.setdefault("fields", []).append(f) + have.add(f.get("key")) + elif lock_fields: + stored = next((g for g in (t.get("fields") or []) + if g.get("key") == f.get("key")), None) + if stored is not None: + # ⭐⭐ WAVE 41 · T22 (R19) — the shipped sentence, and ONLY on a column no + # human has taken custody of. Read BEFORE the `preset` stamp below, so the + # custody question is asked of the field exactly as it was found. + due = _preset_description_due(stored, f) + if due is not None: + stored["description"] = due + automation = dict(stored.get("automation") or {}) + if flow_tag: + automation.setdefault("flowId", str(flow_tag)) + automation["preset"] = True + stored["automation"] = automation + # ADOPTION: a machine name is not an owner. It never overwrites a human one. + if human and (t.get("createdBy") or "") in MACHINE_OWNERS: + t["createdBy"] = human + # An automation's table SAYS an automation owns it — the nav badge reads from this. + t["source"] = t.get("source") or "Automation" + if record_mode: + t["recordMode"] = record_mode + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return key + + +def ut_write_rows(rt, key, rows): + """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is not None: + t["rows"] = rows + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") + + +def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): + """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is not None: + t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value + for k, rws in (extra_rows or {}).items(): + tt = cur.get(k) + if tt is not None: + tt["rows"] = rws + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") + + +IG_FIELD_DESCRIPTIONS = { + "alt_text": "Accessibility text attached to the Instagram post.", + # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would + # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED + # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off + # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same + # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a + # description that named only the first would have quietly excluded the second. + "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " + "rather than measured at the time shown.", + "avg_comments_12": "Average comments across the 12 most recent captured posts.", + "avg_engagement": "Average engagement rate reported for the profile.", + "avg_likes_12": "Average likes across the 12 most recent captured posts.", + "avg_plays_12": "Average plays across the 12 most recent captured posts.", + "avg_views_12": "Average views across the 12 most recent captured posts.", + "views": "View count for the post, as Instagram displays it.", + "video_duration": "Length of the video in seconds.", + "comments_disabled": "Checked when the creator has turned comments off for this post.", + "plays": "Times the video started playing, including replays.", + "bio": "Biography shown on the Instagram profile.", + "bio_hashtags": "Hashtags listed in the profile biography.", + "business_category": "Instagram's business category for the account.", + "caption": "Caption published with the Instagram post.", + "category": "Instagram's category for the profile.", + "comment_key": "Unique ID for the captured comment record.", + "commented_at": "Date the comment was posted.", + "comments": "Comment count reported for the post.", + "comments_captured": "Number of distinct comment records linked to this profile.", + "comments_link": "Comment records linked to this record.", + "country_code": "Country code reported for the profile.", + "created_by": "User whose automation first added this lead.", + "enriched_at": "Date the profile was last enriched.", + # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number + # beside it is the only thing that says how much of one. + "location_guess": "Most common city tagged across this profile's captured posts - a guess, " + "not a stated location.", + "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " + "city.", + "external_url": "Website linked from the Instagram biography.", + "external_url_title": "Title Instagram shows for the biography link.", + "fbid": "Facebook ID associated with the Instagram account.", + "first_found": "Date an automation first found this profile.", + "followers": "Latest reported follower count.", + "following": "Latest reported number of accounts followed.", + "found_count": "Number of times discovery automations found this profile.", + "full_name": "Display name shown on the Instagram profile.", + "handle": "Instagram username without the @ symbol.", + "has_channel": "Checked when the profile has an Instagram channel.", + "hashtags": "Hashtags extracted from the post caption.", + "highlights_count": "Total story highlight collections reported for the profile.", + "ig_id": "Instagram's internal ID for the account.", + "influencer_key": "Normalized handle linking this row to its profile.", + "is_business": "Checked when Instagram marks the account as a business.", + "is_joined_recently": "Checked when Instagram marks the account as recently joined.", + "is_private": "Checked when the Instagram profile is private.", + "is_professional": "Checked when Instagram marks the account as professional.", + "last_found": "Date an automation most recently found this profile.", + "likes": "Like count reported for this record.", + "measured_at": "Date engagement metrics on this post were last read.", + "measurements_captured": "Number of measurement rows stored for this post.", + "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", + "partner": "Brand named in the post's paid partnership metadata.", + "partner_id": "Partner ID reported for the Instagram profile.", + "platform": "Social network for this profile.", + "plays": "Video plays, including repeat plays, reported for the post.", + "post_link": "Post record linked to this measurement or comment.", + "post_measurements_captured": "Number of distinct post measurements linked to this profile.", + "post_snapshot_key": "Unique ID for this post measurement.", + "post_snapshots_link": "Engagement measurement rows linked to this record.", + "posted_at": "Date the Instagram post was published.", + "posts_captured": "Number of distinct post records captured for this profile.", + "posts_count": "Total posts reported for the Instagram profile.", + "posts_link": "Post records linked to this profile.", + "profile_name": "Profile name returned by the data provider.", + "profile_reads": "Number of profile snapshots stored for this profile.", + "profile_snapshots_link": "Profile snapshot rows linked to this profile.", + "profile_url": "Direct URL to the Instagram profile.", + "pronouns": "Pronouns shown on the Instagram profile.", + "pulled_at": "Date this snapshot was collected.", + "related_accounts": "Accounts Instagram suggests alongside this profile.", + "replies": "Reply count reported for the comment.", + "shortcode": "Instagram shortcode that uniquely identifies the post.", + "source_payload": "Full source response for this record.", + "snapshot_key": "Unique ID for this profile snapshot.", + "source": "Method or provider used to read this profile.", + "tagged_location": "Location tagged on the post; it is not the creator's residence.", + "text": "What the comment says. The commenter's name is not stored as a column.", + "type": "Instagram media type: image, video, or carousel.", + "url": "Direct URL to the Instagram post.", + "verified": "Checked when Instagram marks the profile as verified.", +} + + +#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the +#: map above is one line of `field_def`: it defaults `description` to +#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column +#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`, +#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram +#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check. +#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting +#: the Instagram sentence is the failure this map exists to prevent. +TT_FIELD_DESCRIPTIONS = { + "platform": "Network this handle is on.", + "handle": "TikTok @name; the unique account handle.", + "full_name": "Display name shown on the TikTok profile.", + "tt_id": "TikTok's own numeric id for the account.", + "profile_url": "Direct URL to the TikTok profile.", + "bio": "Profile biography text.", + "external_url": "Link in the TikTok bio.", + "verified": "Checked when TikTok marks the account as verified.", + "is_private": "Checked when the account is private.", + "followers": "Follower count at the time of the pull.", + "following": "Accounts this profile follows.", + "posts_count": "Videos published by this account.", + "likes_received": "Total likes this account's videos have received.", + "avg_engagement": "Average engagement rate, stored as a percentage.", + "comment_engagement": "Comment engagement rate, stored as a percentage.", + "like_engagement": "Like engagement rate, stored as a percentage.", + "is_business": "Approximate: set when TikTok flags the account as a commerce user.", + "country_code": "Two-letter country code reported for the account.", + "predicted_lang": "Language TikTok predicts for this account.", + "account_created_at": "When the TikTok account itself was created.", + "region": "Region reported for the account.", + "shortcode": "TikTok's numeric id for the video; unique per post.", + "type": "TikTok post type: video or image.", + "url": "Direct URL to the TikTok post.", + "caption": "Post description text.", + "posted_at": "When the creator published the post.", + "likes": "Likes reported for the post.", + "comments": "Comment count reported for the post.", + "views": "Play count TikTok reports for the video.", + "shares": "Times the post was shared.", + "saves": "Times the post was saved to a collection.", + "video_duration": "Video length in seconds.", + "hashtags": "Hashtags used in the post.", + "tagged_location": "Commerce location reported for the post; it is not the creator's home.", + "influencer_key": "Handle of the account this record belongs to.", + "comment_key": "Unique ID for this comment.", + "commented_at": "When the comment was posted.", + "text": "What the comment says. The commenter's name is not stored as a column.", + "replies": "Reply count reported for the comment.", + "snapshot_key": "Unique ID for this profile snapshot.", + "post_snapshot_key": "Unique ID for this post measurement.", + "pulled_at": "When this measurement was read.", + "enriched_at": "When this row was last enriched.", + "source": "Method or provider used to read this record.", + "source_payload": "Full source response for this record.", +} + + +def field_def(text_key, label, ftype="text", **extra): + """One machine-spawned column definition. + + ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, + `profile`), and it stays ONE constructor rather than growing a second for "special" fields. + ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s + W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole + preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). + + ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a + feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so + `core.user_tables._clean_field` — the single validator every field created through the ordinary + door passes — has never judged a single field this engine spawned. The difference was exactly + one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` + for every automation column in the product. + ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an + absent key was already not that, so the bypass was fail-closed and therefore silent. It is + stated now, and `verify_automation` asserts the equality field-by-field, so the next key + `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). + """ + description = " ".join(str(extra.pop( + "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) + out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", + "editRole": "admins"} + if description: + out["description"] = description + out.update(extra) + return out + + +def tt_field_def(text_key, label, ftype="text", **extra): + """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2). + + ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can + never be reached from here. An explicit empty string makes `field_def` omit the key, which is + the same shape an undescribed column already has — the point is that the sentence a TikTok + column carries is one somebody wrote about TikTok, or none at all. + """ + extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, "")) + return field_def(text_key, label, ftype, **extra) + + +# --------------------------------------------------------------------------------------------- +# DEFINITIONS — validation and the bucket's read/write half +# --------------------------------------------------------------------------------------------- + +def _s(v, n=200): + return str(v if v is not None else "")[:n] + + +#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. +#: +#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 +#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config +#: disagree with what the run can possibly do. +#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect +#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve +#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is +#: worse than one that refuses it, because the refusal is the only place the ceiling can be +#: taught. So this REFUSES, and the sentence names the cap and who set it. +#: +#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the +#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the +#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask +#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` +#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a +#: property of the call we happened to make. +#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of +#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE +#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to +#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it +#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. +MAX_POSTS_PER_PULL = 30 +DEFAULT_POSTS_PER_PULL = 10 + +#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately +#: NOT `MAX_POSTS_PER_PULL`. +#: +#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap +#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so +#: raising the capture cap to 30 silently made every one of them average thirty posts under a +#: label that says twelve. Nobody would have looked at those columns again to check. +#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different +#: decisions, and only the first one was made. To widen the average, change this constant AND the +#: four keys and labels together — a column whose meaning changes underneath its own name is +#: worse than one that is merely narrow. +AVG_WINDOW_POSTS = 12 + + +#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a +#: filter cannot ask for a category that can never match a stored row. +POST_TYPES = ("video", "image", "carousel") +#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason +#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video` +#: is the stored key: a client that translated it locally would be a second copy of this +#: vocabulary, free to drift the day a fourth kind appears or a name changes. +#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a +#: label naming only reels would over-promise on a plain video post. +POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"} + + +def clean_post_groups(raw, max_posts): + """`(groups, error)` for `config.postGroups` — "last N reels, last M images". + + ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: + `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. + + ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today + carries no such key, and inventing a filter for it would silently start DROPPING posts those + automations have always captured. Empty list and missing are the same answer. + + ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the + difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them + would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken + one, so the only way to see one is a person typing it, and a filter that silently ignores the + type you asked for is how you end up paying for reels and storing carousels. + """ + if raw in (None, "", [], {}): + return [], None + if not isinstance(raw, list): + return None, "the post groups have to be a list of {type, limit} entries" + out, seen = [], set() + for item in raw: + if not isinstance(item, dict): + return None, "each post group has to be a {type, limit} entry" + t = str(item.get("type") or "").strip().lower() + if t not in POST_TYPES: + return None, (f"'{t or 'blank'}' is not a post type. Use one of " + f"{', '.join(POST_TYPES)}") + if t in seen: + return None, f"the post groups name '{t}' twice. One limit per type" + seen.add(t) + try: + lim = int(item.get("limit")) + except (TypeError, ValueError): + return None, f"how many {t} posts to keep has to be a whole number" + if lim < 1: + return None, f"a {t} group has to keep at least one post" + # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures + # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is + # refused HERE, where the number they typed is still on the screen in front of them. + if lim > max_posts: + return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " + f"group cannot keep {lim}. Raise the post count first") + out.append({"type": t, "limit": lim}) + return out, None + + +def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): + """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. + + ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this + takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old + clamp's default, which this cap now forbids. If an inherited value were refused the same way + a typed one is, every one of those automations would 400 on its next Save **forever**, for a + number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator + that refuses stored data does not protect the user from it, it locks them out of their own + automation. + So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely + INHERITED gets silently brought inside the cap it was already effectively subject to at the + vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default, None + try: + n = int(raw) + except (TypeError, ValueError): + if not submitted: + return default, None + return None, "the post limit must be a whole number" + if not submitted: + return max(1, min(n, MAX_POSTS_PER_PULL)), None + if n < 1: + return None, "keep at least one post per profile, or turn the post capture off" + if n > MAX_POSTS_PER_PULL: + return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts. That is the " + f"vendor's cap, not ours, so {n} would store the same " + f"{MAX_POSTS_PER_PULL} and read as more") + return n, None + + +def clean_config(kind, raw, previous=None): + """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. + + ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail + rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not + edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" + from a panel that never knew about them would silently turn off the dry run, or turn ON a + per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; + present ⇒ take it, including `false`. + """ + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + + def flag(name): + return bool(raw[name]) if name in raw else bool(prev.get(name)) + if kind == "plain": + # A plain automation has no machine step, so it only needs the database its records walk. + # + # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on + # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- + # trigger — so requiring a second copy of that fact here would be a mandatory field with + # exactly one legal answer, and a Save that refuses until you retype what you just picked. + # There is no `dryRun`: dry-run means "run the machine step but do not write", and there + # is no machine step to run. + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + return {"targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None + if kind == "scrape_db": + url = _s(raw.get("url"), 2000).strip() + if not url: + return None, "a source URL is required" + try: + guard(url) + except Refused as e: + return None, str(e) + extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" + fmap = {} + for k, v in list((raw.get("fieldMap") or {}).items())[:60]: + tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") + if tk: + fmap[_s(k, 120)] = tk[:60] + if not fmap: + return None, "map at least one source column to a field" + key_field = _s(raw.get("keyField"), 60).strip() + if key_field not in fmap.values(): + return None, "the key field must be one of the mapped fields" + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + return {"url": url, "extract": extract, + "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), + "fieldMap": fmap, "keyField": key_field, + "targetTable": target, + "dryRun": flag("dryRun"), + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None + if kind == "field_instagram": + table = _s(raw.get("targetTable"), 60).strip() + if not table.startswith(UT_PREFIX): + return None, "an Instagram automation runs against a blank database (ut_*)" + fkey = _s(raw.get("fieldKey"), 80).strip() + if not fkey: + return None, "pick the automation column this run writes into" + # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). + # They are read from nothing and written to nothing: a stored definition carrying either + # still SAVES — it simply loses them on the next write — and neither is ever a reason to + # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown + # key would 400 every automation a tenant stored before this wave, forever, on a screen + # that gives them no way to remove it. Dropping a retired key is a migration; refusing it + # is an outage. + # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary + # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired + # USER concept — renaming them would be a second, unrelated change wearing this one's + # justification. + # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel + # that does not edit the post count must not reset it to the default on Save. And an + # INHERITED value is clamped rather than refused; see `clean_max_posts`. + sent = "maxPosts" in raw + max_posts, perr = clean_max_posts( + raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) + if perr: + return None, perr + return {"targetTable": table, "fieldKey": fkey, + "urlField": _s(raw.get("urlField"), 80).strip(), + # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked + # for: the Profiles row carries post IDENTITY for free but NO engagement + # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. + "postMetrics": flag("postMetrics"), + # The full Comments dataset can bill many rows per post. It is an explicit, + # independent opt-in and never follows the per-post switch automatically. + "commentMetrics": flag("commentMetrics"), + "dryRun": flag("dryRun"), + "maxPosts": max_posts}, None + # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor + # the same question of two different corpora: a record ceiling, a predicate list, a join word + # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the + # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write + # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists + # to prevent, and it would do it silently. + if kind in DISCOVERY_KINDS: + # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now + # come from `discovery_facts` rather than being spelled out here. They were correct; they + # were also the FIFTH copy of "which table does this kind write to", and the other four + # were the ones wave 29 forgot to update. + _, _disc_table, _disc_label, _ = discovery_facts(kind) + limit = int(raw.get("recordsLimit") or 0) + if limit < 1: + return None, ("say how many profiles to fetch. An UNBOUNDED discovery query is the " + "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") + if limit > BD_MAX_RECORDS: + return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" + # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what + # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading + # the operator afterwards is how the OR hole survived: the guard was handed the branches + # and never told they were a union. + join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" + # ⭐ WAVE 32 · T46 (D-167) — AND THE KIND GOES WITH THEM. `clean_config` already knows which + # corpus this automation searches; passing it is what stops a `discover_tiktok` being built + # on the 16 Instagram fields TikTok's dataset does not carry — a search that does not error, + # returns nothing, and reads as "no such creators exist" after the money is spent. + preds, perr = clean_predicates(raw.get("predicates"), join, kind) + if perr: + return None, perr + target = _s(raw.get("targetTable"), 60).strip() or _disc_table + if not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) + if serr: + return None, serr + return {"recordsLimit": limit, "predicates": preds, + "operator": join, + "targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, + # C6/R5: WHERE the conditions above came from, when they were derived. Stored so + # the surface can say "these were filled in from the 'Florists' view, over 12 + # records" instead of presenting them as if somebody typed them. + "seed": seed, + "dryRun": flag("dryRun")}, None + return None, f"unknown automation kind {kind!r}" + + +def clean_seed(raw): + """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter + somebody typed by hand has no seed, and that is the ordinary case. + + ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that + the derived conditions are "written into `config.predicates` as ordinary conditions — it is a + filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, + like every other search. Keeping it means the surface can say where those rows came from, and + a user editing them freely is exactly what is supposed to happen. + """ + if raw in (None, "", {}): + return {}, None + if not isinstance(raw, dict): + return None, "the seed must be an object" + source = _s(raw.get("source"), 12).strip().lower() + if source not in SEED_SOURCES: + return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) + table = _s(raw.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, f"a seed reads a blank database (ut_*). {table!r} is not one" + basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} + derived, _err = clean_predicates(raw.get("derived") or [], "and") + return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), + # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what + # was suggested; the conditions that matter are already in `predicates` and were + # validated there. Refusing a Save because a stored provenance note aged badly would + # block the user from editing the very filter it describes. + "derived": derived or [], + "basis": {"rows": max(0, int(basis.get("rows") or 0)), + "fields": [f for f in (basis.get("fields") or []) + if isinstance(f, dict)][:SEED_MAX_PREDICATES], + "related": basis.get("related") + if isinstance(basis.get("related"), dict) else {}, + "note": _s(basis.get("note"), 200)}}, None + + +def clean_schedule(raw, previous=None): + """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` + measures from — without it, enabling a daily job fires it immediately).""" + raw = raw if isinstance(raw, dict) else {} + cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" + parse_cron(cron) # raises -> the route answers 400 + enabled = bool(raw.get("enabled")) + prev = previous or {} + out = {"cron": cron, "enabled": enabled} + if enabled: + out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() + return out + + +# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── +# The owner's words: "we can make this automation a loop, so the ending should always be defined, +# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, +# or the user have to click a button to reset, or after a certain amount of time it can +# automatically reset to first cycle." +# +# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition +# that predates this field must not start moving records on its own the day the code ships. A +# loop is a thing somebody turns on. +def clean_flow(raw, previous=None, notes=None, rt=None): + """Validate the builder's ordered action list. `(flow, error)`. + + ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. + """ + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + actions, err = clean_actions( + raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) + if err: + return None, err + return {"actions": actions}, None + + +#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the +#: definition carries none. +#: +#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at +#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the +#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly +#: created automation. Two numbers for one fact, in the panel, before anybody had typed +#: anything. **Read off the screenshot; every assertion in the battery was green.** +#: +#: This is the same species as the default-versus-guard split fixed the same day: a value +#: decided in one file and a value decided in another, describing the same thing. Aligning the +#: constants closes the only window in which they can disagree — the seed — because every later +#: state comes from the stored config. +DISCOVER_SEED_RECORDS = 25 + + +def discovery_seed_spec(kind): + """-> (default table, enrich action kind) for a discovery KIND. + + ⭐⭐ WAVE 30, OWNER REPORT 2026-08-12, verbatim: *"'When a Tiktok profile fits a criteria' + should ALWAYS have a 'Create record' EXACTLY like the Instagram one … THE ONLY DIFFERENCE IS + THE COLUMNS AND DATA SCHEMA"*. They were right, and the seeding path was the one family of + sites this wave widened everywhere ELSE: `clean_definition` planted step 1 and step 2 only when + the trigger was `ig_profile_match`, so a TikTok search stored `flow.actions = []` and had + nowhere to put what it found — MEASURED live against `auto_1` (Instagram: create_record + + enrich) versus a fresh TikTok discovery (empty). + + ⭐ THIS IS A LOOKUP, NOT A SECOND SEEDER, and that is the whole point of the ruling. One + builder plants both platforms' steps; the only things that vary are the table the record lands + in and which enrich action reads it — literally "the columns and data schema". A forked + `_ensure_tt_action` would start identical and drift, which is the failure `ENRICH_KINDS` and + `DISCOVERY_KINDS` were introduced to prevent one screen up. + + ⚠ Resolved in a FUNCTION rather than a module-level dict because `DISCOVER_TABLE` is defined + far below this line; a dict here would raise at import. + """ + if kind == "discover_tiktok": + return TT_PROFILE_TABLE, "enrich_tiktok" + return DISCOVER_TABLE, "enrich_instagram" + + +def _ensure_ig_action(flow_raw, table, enrich_kind="enrich_instagram"): + """Create record is PERMANENT step 1 on a DISCOVERY trigger — it cannot be deleted or moved. + + ⚠ THE NAME IS INSTAGRAM'S AND THE BEHAVIOUR IS BOTH PLATFORMS' (wave 30). Renaming it would + churn six gate references for no behaviour change; `enrich_kind` is what makes it general, and + it defaults to Instagram's so every pre-existing caller keeps its exact meaning. + + ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, + on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a + comment warning that re-seeding on every clean would be "a control that will not take no for + an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be + deleted, because the nature of that automation is that it needs to first create a record in a + database somewhere from the list of profiles to fetch."* + + Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER + produces rows has nowhere to put them until something writes them, so an Instagram search + with no Create record is not a customised automation — it is a search whose results are + discarded. That is not a preference to respect. + + ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and + the values it maps are theirs. An existing `create_record` further down is MOVED to the front + rather than duplicated — re-seeding a second one would quietly double every run's writes. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + if actions and actions[0].get("kind") == "create_record": + return _pin_unique(_ensure_enrich_step( + flow_raw if isinstance(flow_raw, dict) else flow, enrich_kind)) + at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) + if at > 0: + actions.insert(0, actions.pop(at)) + else: + # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner + # and no card: the wave-23 header records four kinds that shipped with illegal seeds and + # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. + actions.insert(0, { + "id": "act_1", "kind": "create_record", "enabled": True, "when": None, + # `config.label` follows the `review` action's precedent in `_clean_action_config` — + # an action naming itself, inside the untyped config bag, so no shared TS type changes. + "config": {"table": str(table or DISCOVER_TABLE), + "label": "Save the profile", + # C5: the picture and the real write finally agree — see `_pin_unique`. + "uniqueOn": "handle", + "values": {"handle": "{{handle}}"}}, + }) + flow["actions"] = actions + return _pin_unique(_ensure_enrich_step(flow, enrich_kind)) + + +def _ensure_enrich_step(flow_raw, enrich_kind="enrich_instagram"): + """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON A DISCOVERY SEARCH. + + ⚠ WAVE 30: `enrich_kind` selects the network. Instagram's is the default so every pre-existing + caller means exactly what it meant before; TikTok passes `enrich_tiktok` and gets the identical + step shape, which is the owner's *"only the columns and data schema differ"*. + + Owner: *"make it default that this enrichment action is Step 2 always, and under Config of + Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for + the same reason: a search that finds profiles and never reads them has done half a job. The + difference is the control — step 1 is permanent because a flow without it discards its + results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and + disabling would be two ways to say one thing, and only one of them survives a re-save. + + ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains + this step by upgrading does not quietly start spending; switching the Source to the paid + provider is an explicit choice a person makes in front of the sentence that names the cost. + ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — + a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow + re-buying the same profile nightly. Off-by-default would make the expensive behaviour the + accident. + + ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an + If/then branch has one; seeding a second at the top level would enrich twice and bill twice, + which is the duplication `apply_actions` already paid for once with the pinned step 1. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + if any(a.get("kind") == enrich_kind for a in walk_actions(actions)): + return flow_raw if isinstance(flow_raw, dict) else flow + seeded = list(actions) + # Index 1 — after the pinned Create record, because there is nothing to enrich until the + # profiles have been written as records. `insert` past the end is a plain append, so a flow + # with only step 1 lands this at the end, which IS step 2. + seeded.insert(1, { + "id": "act_enrich", "kind": enrich_kind, "enabled": True, "when": None, + "config": {"postMetrics": False, "commentMetrics": False, + "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, + "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", + "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, + "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, + }) + return {**flow, "actions": seeded} + + +#: The key the discovery writer really upserts on. Both discovery runners key candidates on +#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's +#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is +#: supplied by the runner, never typed by a person. +#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that +#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is +#: the pair above and the TENANT is the unit, while `created_by` survives as an informational +#: "Found by" stamp that no run may branch on. +#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note +#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate +#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a +#: bug with a rationale attached. +IG_PINNED_UNIQUE = "handle" + + +def _pin_unique(flow): + """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. + + ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — + the builder posts the whole action list on every save — so a client that omitted the key would + silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's + own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming + "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this + module refuses everywhere else. + + ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason + this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action + does not write, so the stamp is applied ONLY when `handle` is among the action's own values. + A user who remaps the card to write different columns keeps their edit and still saves; the + alternative — stamping unconditionally — is a product-seeded default that its own validator + would 400, which is precisely the post-W24 hotfix this rule exists because of. + + ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes + `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED + definition's own dict, so writing into it would edit the live store object in memory, before + (and regardless of) any commit. Every touched level is copied instead. + """ + if not isinstance(flow, dict): + return flow + acts = flow.get("actions") + if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): + return flow + first = acts[0] + if first.get("kind") != "create_record": + return flow + cfg = first.get("config") + if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): + return flow + if str(cfg.get("uniqueOn") or "").strip(): + return flow + return {**flow, + "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} + + +#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so +#: the seeder and the un-seeder cannot disagree about what "seeded" means. +#: ⚠ WAVE 30 — `enrich_tiktok` shares `act_enrich`. The map is keyed by KIND, and only one enrich +#: step is ever seeded per flow (the network follows the trigger), so the id cannot collide. +_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich", + "enrich_tiktok": "act_enrich"} + + +def _is_untouched_ig_seed(action, table, enrich_kind="enrich_instagram"): + """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) + + ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and + that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build + the same dicts one screen above, so a step gaining a config key next wave gains it here too. + A remembered key list would quietly start calling every seeded action "edited". + + ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The + stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the + kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW + dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", + and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only + version where the two sides are the same kind of object. + + ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the + TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about + a step they still want, and an off seed is still a seed. + """ + if not isinstance(action, dict): + return False + kind = str(action.get("kind") or "") + if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): + return False + seeded, _seed_err = clean_actions( + (_ensure_ig_action({"actions": []}, table, enrich_kind).get("actions") or [])) + fresh = {a.get("kind"): a for a in (seeded or [])} + seed = fresh.get(kind) + if not seed: + return False + cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) + if kind in ENRICH_KINDS: + cfg.pop("enabled", None) + seed_cfg.pop("enabled", None) + return cfg == seed_cfg and (action.get("when") or None) is None + + +def _drop_ig_seeds(flow_raw, table, enrich_kind="enrich_instagram"): + """Strip the trigger's own seeded steps, keeping any the person has since made theirs. + + ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT + THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it + does not delete the seeded actions … a create_record the person has since re-pointed at their + own table with their own values is THEIR action now"*. That reasoning is still correct and is + exactly what this function preserves — but it was applied to EVERY seeded action, including + the ones nobody had ever opened, and the result is what the owner reported: switch an + Instagram search to Manual and you are left holding a "Save the profile" step writing + `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not + somebody's work being protected; it is the machine's own leftover, pointed at a table the + automation has nothing to do with any more. + + ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as + `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a + branch that destroys something, so the conservative half is the load-bearing half. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + kept = [a for a in actions if not _is_untouched_ig_seed(a, table, enrich_kind)] + if len(kept) == len(actions): + return flow_raw + return {**flow, "actions": kept} + + +def ig_action_pinned(defn, index): + """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` + flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" + # ⭐ WAVE 30 — BOTH discovery kinds. It read `== "discover_instagram"`, so TikTok's step 1 (once + # seeded) would have rendered as an ordinary draggable, deletable card: the same rule the owner + # asked for "EXACTLY", applied to only one network. + return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 + + +def clean_definition(raw, previous=None, username="", notes=None, rt=None): + """Whole-definition validation. Returns `(defn, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous or {} + # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing + # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second + # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys + # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload + # invalid in BOTH its trigger and its config now answers with the trigger's sentence. + trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw + else prev.get("trigger"), prev.get("trigger")) + if terr: + return None, terr + # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this + # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a + # dead end the moment the wizard was deleted and the client's create body became {name}. + kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND + drop_ig_seeds = False + if (trigger or {}).get("key") == "ig_profile_match": + # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather + # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and + # whose kind says something else is not a preference to respect, it is two halves of one + # answer disagreeing, and the trigger is the half the person actually picked. + kind = "discover_instagram" + elif (trigger or {}).get("key") == "tiktok_profile_match": + # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how + # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram + # arm is — a definition whose trigger says TikTok discovery and whose kind says something + # else is two halves of one answer disagreeing. + kind = "discover_tiktok" + elif (kind == "discover_tiktok" + and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match" + and (trigger or {}).get("key") != "tiktok_profile_match"): + # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm + # below): with nothing to flip the kind BACK, switching the trigger to Manual left + # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus + # search on a flow the person had just made manual.** Shipped here in the same change as + # law 1 rather than discovered the same way twice. + # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the + # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because + # `patch()` builds its raw as `dict(prev)` plus the caller's keys. + kind = DEFAULT_KIND + # ⭐ WAVE 30 — un-seed on the way out, exactly as the Instagram arm below does. This line + # was ABSENT and harmless for as long as TikTok had no seeds to leave behind; the moment + # the seeder above learned TikTok, its absence became wave-27 item 15's defect on the other + # network — a flow switched to Manual still carrying a "Save the profile" step nobody + # planted deliberately. Found by widening the seeder and asking what else assumed only + # Instagram could have seeds. + drop_ig_seeds = True + elif (kind == "discover_instagram" + and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" + and (trigger or {}).get("key") != "ig_profile_match"): + # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. + # + # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram + # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch + # the trigger to Manual and the automation stayed `discover_instagram`, which means + # + # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow + # the person had just made MANUAL fired a PAID Bright Data corpus search; + # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed + # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the + # instagram trigger, the 'always first' create record just stuck there"*. The server + # would have accepted the delete; the CLIENT hid the control, because both halves ask + # the kind and the kind was lying. + # + # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in + # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as + # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY + # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. + # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` + # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest + # question: did this automation stop being an Instagram search? + # + # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram + # trigger — a `discover_instagram` created with an explicit kind and some other trigger is + # somebody's deliberate state, not a mistake to correct. And only FROM + # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose + # triggers are their own business, and clobbering them would retire them by accident. + # + # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this + # replaces argued the other way, so here is why it was half right. It read: *"it does not + # delete the seeded actions … a create_record the person has since re-pointed at their own + # table with their own values is THEIR action now, and quietly destroying it is the silent + # data loss this module refuses everywhere else."* Every word of that is still true and is + # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the + # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to + # Manual still carrying a "Save the profile" step writing `{{handle}}` into + # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces + # handles, unpinned but still there, still runnable, and still pointed at a table that has + # nothing to do with this automation. + # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. + kind = DEFAULT_KIND + drop_ig_seeds = True + if kind not in KINDS: + return None, f"unknown automation kind {kind!r}" + # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. + # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` + # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. + # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way + # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two + # live automations save through this function every time their owner edits them. The gate's + # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved + # from `create` into `clean_definition` (the tempting simplification), this helper would go + # red across a dozen sections, which is the alarm that change deserves."* It did, and the + # alarm worked. + # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every + # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody + # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an + # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored + # ones are rewritten under their owner — and that is a ruling, not a refactor. + name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) + if not name: + return None, "name the automation" + cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") + if kind in DISCOVERY_KINDS and prev.get("kind") != kind: + # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS + # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a + # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an + # UNBOUNDED discovery query is the one shape the vendor refuses outright + # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the + # owner saw as "the Trigger won't load". + # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends + # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` + # at that moment, so it falls through to `return { targetTable }`). IG worked only because + # this seed caught it. So the defect was never "TikTok is missing something Instagram has" + # — it was one hard-coded string in the single line that rescues both. + # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old + # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a + # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). + # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very + # first Save after picking this trigger 400s — against the stored-inert-with- + # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). + # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard + # exists because an unbounded query is the one shape the vendor refuses outright, and + # coercing a blank into a number would be exactly the silent widening it protects against. + # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until + # a person presses Run now or arms the schedule. + cfg_raw = dict(cfg_raw or {}) + if not _ig_int(cfg_raw.get("recordsLimit")): + cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS + config, err = clean_config(kind, cfg_raw, prev.get("config")) + if err: + return None, err + if trigger: + # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own + # configuration IS the config, and `clean_trigger` could not see it. + trigger["configured"] = _trigger_configured(trigger, config) + try: + schedule = clean_schedule( + raw.get("schedule") if isinstance(raw.get("schedule"), dict) + else prev.get("schedule"), prev.get("schedule")) + except ValueError as e: + return None, str(e) + # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) + flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") + # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only + # call was what made the action deletable; running it on every save is what makes step 1 + # permanent, and it is deliberate rather than a widened condition nobody noticed. + # ⭐⭐ WAVE 30 (owner, 2026-08-12) — BOTH DISCOVERY TRIGGERS SEED, not just Instagram's. This + # single `==` was the whole of *"why is it not copied EXACTLY"*: a TikTok search stored an + # EMPTY flow, so the product's own rule — a search that finds profiles must have somewhere to + # put them — held for one network and not the other. + if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: + _seed_table, _seed_enrich = discovery_seed_spec(kind) + flow_raw = _ensure_ig_action( + flow_raw, config.get("targetTable") or _seed_table, _seed_enrich) + elif drop_ig_seeds: + # ITEM 15: the trigger just STOPPED being a discovery one. Un-seed what the trigger + # planted and nobody has since made theirs — measured against the table the seeds were + # planted for, which is the PREVIOUS config's, not the one being saved. + # ⚠ And against the PREVIOUS kind's enrich action, for the same reason: the seeds to strip + # are TikTok's if that is what was planted, and asking "is this Instagram's seed?" of a + # TikTok flow answers no and silently leaves the leftover behind. + _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) + flow_raw = _drop_ig_seeds( + flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) + flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) + if ferr: + return None, ferr + # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that + # its `config.table` is authoritative has to reach it, or the two fields that name one database + # would disagree on exactly the network that just gained the card. + if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: + # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and + # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since + # wave 24: the pinned step 1 says where profiles are written, and the config says where + # the automation's records live — and for a discovery flow those are the same database by + # construction. When they disagreed, every surface picked a different winner (the canvas + # Write node read the config, the actual write read the action), which is a bug you can + # only see by comparing two panels. + # + # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the + # action's own configuration, so the table named there is the one they were looking at. + # + # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: + # a stored definition carrying two different values is reconciled on its next clean, + # silently and exactly once, because nothing writes the losing value back. A refusal here + # would 400 every Save of a definition that was legal yesterday. + # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the + # TIE-BREAK rather than the whole rule. The owner's report: change the database on the + # TRIGGER, save, and the picker springs back to the old one with no message. The cause is + # the branch below read unconditionally — the pinned action still named yesterday's table, + # so it overwrote a change the person had just made in front of it, silently, every time. + # + # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled + # "when the two disagree, whose value is real?" — the action's, because R2b puts the + # preset list on the action's own panel, so that is the one they were looking at. That + # answers a definition at REST. A save is different: it carries an INTENT, and the honest + # question is which side this particular save changed. So: + # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to + # follow it (leaving the action behind would just re-revert on the next save); + # · only the action moved, or neither did and they were already inconsistent -> R2's + # migration, unchanged, reconciling a stored definition exactly once. + # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is + # also the only reading that cannot lose a value: the action's table is the one with + # the field mapping attached to it. + first = (flow.get("actions") or [{}])[0] + pinned = str((first.get("config") or {}).get("table") or "") \ + if first.get("kind") == "create_record" else "" + prev_target = str((prev.get("config") or {}).get("targetTable") or "") + prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] + ).get("config") or {}).get("table") or "") + target_moved = bool(prev_target) and config.get("targetTable") != prev_target + action_moved = bool(prev_pinned) and pinned != prev_pinned + if pinned and target_moved and not action_moved: + # The person changed the trigger's database. Carry it INTO the pinned step so the two + # agree, instead of throwing their edit away to keep a stale copy consistent. + cfg1 = dict(first.get("config") or {}) + cfg1["table"] = str(config.get("targetTable") or "") + flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] + elif pinned and pinned != config.get("targetTable"): + config["targetTable"] = pinned + # The stored label described a DIFFERENT database, so keeping it would caption the new + # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` + # never relabels a table that already exists — so an adopted hand-made database keeps + # the name its owner gave it. + config["targetLabel"] = "" + return { + "id": _s(prev.get("id") or raw.get("id"), 40), + "name": name, "kind": kind, "config": config, "schedule": schedule, + "trigger": trigger, + # The Builder's ordered actions. An absent flow is simply an automation with no actions. + "flow": flow, + "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, + "runs": list(prev.get("runs") or [])[:MAX_RUNS], + # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 + # minutes to build, so a run persists its id here and the NEXT run collects it. It is + # carried through `patch` untouched because a user pressing Save must not silently orphan + # a set the vendor is already building (and already counting against the funds gate). + "state": dict(prev.get("state") or {}), + "created": prev.get("created") or _iso(), + "createdBy": prev.get("createdBy") or username, + # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. + # + # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. + # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded + # system agent undeletable — and this function is a WHITELIST that did not carry the key, so + # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The + # seed would then mint it again on the next list call: a delete that appears to work and + # undoes itself. Found by asserting the round trip rather than by reading the code. + # + # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST + # `{"system": "anything"}` and mint themselves an automation nobody can delete — a + # privilege escalation through a field nobody validates. The flag is granted by the seeder + # and inherited from the stored row, never asserted by a payload. + # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already + # describes as "sticky across a PATCH". One idea, one behaviour, two objects. + **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), + }, None + + +def set_state(rt, auto_id, patch_state): + """Merge into ONE automation's engine state. A key set to None is removed. + + Its own tiny store write rather than a field on `_commit_run`, because the handoff must + survive a run that ends in `error` — a snapshot the vendor is building does not stop existing + because the run that started it failed afterwards. + """ + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is None: + return cur + st = dict(d.get("state") or {}) + for k, v in (patch_state or {}).items(): + st.pop(k, None) if v is None else st.__setitem__(k, v) + d["state"] = st + return cur + + # ⚠ W31-T35 — THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a + # trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here + # would re-read the whole bucket one row later than before and undo D-134 entirely. Safe + # because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write + # into the memo in the same statement. Read `_store_update`'s note before adding a second. + _store_update(rt, _up, flush="sync", keeps_defs=True) + + +def _without_retired_board(definition): + """Return a definition with retired Board-only state removed, without mutating the store. + + This protects scheduled/manual execution before the next UI list request persists the + migration. It removes only the former Board configuration, actions, and audit trail; normal + actions and all user data remain intact. + """ + if not isinstance(definition, dict): + return definition, False + out, changed = dict(definition), False + cfg = out.get("config") + if isinstance(cfg, dict) and "lanes" in cfg: + cfg = dict(cfg) + cfg.pop("lanes", None) + out["config"] = cfg + changed = True + + def _actions(actions): + nonlocal changed + clean = [] + for action in actions if isinstance(actions, list) else []: + if not isinstance(action, dict): + clean.append(action) + continue + if action.get("kind") == "review": + changed = True + continue + item = dict(action) + if item.get("kind") == "group" and isinstance(item.get("config"), dict): + group_cfg = dict(item["config"]) + branches = group_cfg.get("branches") + if isinstance(branches, list): + kept = [] + for branch in branches: + if not isinstance(branch, dict): + changed = True + continue + next_actions = _actions(branch.get("actions")) + if not next_actions: + changed = True + continue + kept.append({**branch, "actions": next_actions}) + if not kept: + changed = True + continue + if kept != branches: + changed = True + group_cfg["branches"] = kept + item["config"] = group_cfg + elif isinstance(group_cfg.get("actions"), list): + nested = _actions(group_cfg["actions"]) + if not nested: + changed = True + continue + if nested != group_cfg["actions"]: + group_cfg["actions"] = nested + item["config"] = group_cfg + clean.append(item) + return clean + + flow = out.get("flow") + if isinstance(flow, dict): + next_flow = dict(flow) + actions = _actions(flow.get("actions")) + if actions != flow.get("actions"): + next_flow["actions"] = actions + if "ending" in next_flow: + next_flow.pop("ending", None) + changed = True + out["flow"] = next_flow + if "reviews" in out: + out.pop("reviews", None) + changed = True + return out, changed + + +def retire_automation_board_state(rt, tables=None): + """Persist the idempotent Board retirement, then remove its generated table fields. + + ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches + the stage scan and nothing else. + """ + fields = retire_automation_stage_fields(rt, tables=tables) + try: + stored = dict(rt.get(STORE_KEY) or {}) + except Exception: + return {"definitions": 0, **fields} + plan = {str(aid): cleaned for aid, raw in stored.items() + for cleaned, changed in [_without_retired_board(raw)] if changed} + if not plan: + return {"definitions": 0, **fields} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for aid, cleaned in plan.items(): + if aid in cur: + cur[aid] = cleaned + return cur + + _store_update(rt, _up, flush="sync") + return {"definitions": len(plan), **fields} + + +#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe. +#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which +#: exactly one caller uses (`grid_hook`). +_DEFS_MEMO = {} +#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a +#: cache — an import of 20,000 rows arrives in far less than this, and a stale trigger for two +#: seconds is bounded and recoverable where a stale one for a minute is a mystery. +_DEFS_TTL = 2.0 + +# The durable external scheduler has a different read pattern from a row-event burst. A live +# Space is one process / one replica today, and every automation-definition write in this module +# passes through `_store_update`, so the scheduler can retain definitions until a REAL write +# invalidates them. This is the cost boundary for Neon: an idle 15-minute wake-up must not turn +# into 96 reads/day of the same tenant JSON value merely because a timer fired. +_TICK_DEFS_MEMO = {} +_TICK_CACHE_LOCK = threading.RLock() + + +def _store_update(rt, fn, flush="sync", keeps_defs=False): + """THE one door every write to the automations bucket goes through — and the ONLY reason it + exists is that the memo's invalidation must be DERIVED rather than remembered. + + ⚠ `keeps_defs=True` IS AN EXPLICIT, SINGLE-SITE OPT-OUT, and it is here because the safe + default would otherwise make the memo useless on the exact path it was built for. `set_state` + writes `state` and NEVER a trigger, and it is called by `grid_hook` itself once per created + record — so clearing on it would mean a 20,000-row import re-read the bucket 20,000 times + anyway, one row later than before. The opt-out is safe because `grid_hook` reads exactly one + state key (`rcHighwater`) and mirrors its own write into the memo in the same statement. + ⛔ The DEFAULT is the safe one, so a thirteenth writer added next wave gets invalidation + without knowing this exists; only a caller that has read this paragraph can opt out. + + ⛔ THE ALTERNATIVE WAS TWELVE CALL SITES. `rt.update(STORE_KEY, …)` appeared twelve times in + this module; hanging an invalidation on each would be a hand-maintained list, and the way that + fails is silent: a thirteenth writer added next wave leaves `grid_hook` firing triggers off a + definition set that no longer exists, with nothing red. Same defect class as `patch`'s + hand-maintained patchable-key list, which this module already documents as *"the silent-drop + seat"* [[a-constant-two-features-share]]. + + ⚠ IT CLEARS THE WHOLE MEMO, not this tenant's row, on purpose: identifying the tenant here + would mean trusting `getattr(rt, 'key')` at a WRITE, and a wrong answer there is a stale + trigger set for another tenant. The memo holds at most a handful of entries and the correct, + boring thing costs nothing. + """ + if not keeps_defs: + _DEFS_MEMO.clear() + with _TICK_CACHE_LOCK: + _TICK_DEFS_MEMO.clear() + return rt.update(STORE_KEY, fn, flush=flush) + + +def all_definitions(rt, cached=False): + """Every automation definition for this tenant. + + ⭐⭐ `cached=True` IS D-134's FIX, AND IT IS OPT-IN FOR A REASON. `grid_hook` runs ONCE PER ROW + EVENT and called this unconditionally, so a 20,000-row import performed 20,000 whole-document + deep copies of the automations bucket — `Store.get` re-serialises on every call, hit or miss, + under the store lock. Every other caller is a request handler that reads it once, so widening + the memo to all of them would trade a real guarantee (a request sees the current store) for + nothing measurable. + + ⛔ THE MEMO IS THE SAME OBJECT ON A HIT, and `grid_hook` MUTATES it deliberately — see the + `rcHighwater` write there. That is not a leak of an implementation detail, it is the fix to the + hazard a naive memo creates: `grid_hook` both READS `state.rcHighwater` and WRITES it through + `set_state`, so a memo that went stale against its own write would re-fire `record_created` + for records it had already handled, breaking A2(4)'s *"once per record EVER — undo-proof"*. + Writes through `_store_update` drop the memo; the one write `grid_hook` makes to its OWN copy + is mirrored into it in the same statement. + """ + if cached: + key = str(getattr(rt, "key", "") or "") + hit = _DEFS_MEMO.get(key) + if hit is not None and (time.monotonic() - hit[0]) < _DEFS_TTL: + return hit[1] + try: + raw = dict(rt.get(STORE_KEY) or {}) + except Exception: + return {} + out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()} + if cached: + _DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out) + return out + + +def tick_definitions(rt): + """Definitions for the external scheduler, cached until `_store_update` changes them. + + Live has one replica, all product writes use `_store_update`, and a restart naturally drops + this process cache. The existing two-second memo remains dedicated to row-event bursts. + """ + key = str(getattr(rt, "key", "") or "") + with _TICK_CACHE_LOCK: + hit = _TICK_DEFS_MEMO.get(key) + if hit is not None: + return hit + fresh = all_definitions(rt) + with _TICK_CACHE_LOCK: + return _TICK_DEFS_MEMO.setdefault(key, fresh) + + +def invalidate_tick_cache(tenant=None): + """Drop scheduler control-plane state after an out-of-process tenant change.""" + with _TICK_CACHE_LOCK: + if tenant is None: + _TICK_DEFS_MEMO.clear() + _TICK_TENANT_CACHE.clear() + else: + _TICK_DEFS_MEMO.pop(str(tenant), None) + + +def _new_id(existing): + n = 1 + while f"auto_{n}" in existing: + n += 1 + return f"auto_{n}" + + +#: C2's three answers to "which database does this automation work on?" — the FIRST question the +#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). +TARGET_MODES = ("existing", "new", "automated") + + +def resolve_target(rt, raw, username=""): + """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. + + - `existing` — bind a database that is already there. + - `new` — mint a blank one now, so the automation has somewhere to write before its first + run instead of conjuring a table the person never agreed to. + - `automated` — leave it to the runner: a scraping automation MINTS its target on first run + (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. + Nothing is created here, deliberately — an empty table created up front for a scrape that + never runs is litter nobody can explain. + """ + if not isinstance(raw, dict) or not raw: + return {}, None # no target block = the pre-wizard shape + mode = _s(raw.get("mode"), 20).strip() or "existing" + if mode not in TARGET_MODES: + return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) + if mode == "automated": + label = " ".join(_s(raw.get("label"), 60).split()) + return ({"targetLabel": label} if label else {}), None + if mode == "existing": + key = _s(raw.get("table"), 60).strip() + if not key: + return None, "choose the database this automation works on" + t = ut_get(rt, key) + if t is None: + return None, f"{key!r} is not a database in this workspace" + return {"targetTable": key, "targetLabel": t.get("label") or key}, None + label = " ".join(_s(raw.get("label"), 60).split()) + if not label: + return None, "name the new database" + import core.user_tables as _ut_new + key = _ut_new.create(label, username or "automation", st=rt) + if not key: + return None, ("the database could not be created. This workspace may be at its table " + "limit") + return {"targetTable": key, "targetLabel": label}, None + + +def _unmint(rt, key): + """Delete a database this call minted moments ago, because the call is refusing (D-50). + + ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user + needs to read; a rollback that raised would replace that sentence with a 500 and lose the + reason the create failed in the first place. The worst case of a swallowed failure here is + the orphan we had before — strictly no worse, and the refusal still reaches the person. + """ + if not key: + return + try: + import core.user_tables as _ut_del + _ut_del.delete(key, st=rt) + except Exception: # noqa: BLE001 + pass + + +def create(rt, raw, username="", notes=None): + existing = all_definitions(rt) + if len(existing) >= MAX_AUTOMATIONS: + return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" + raw = dict(raw or {}) + # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation + # happens after it, so a correct refusal ("a source URL is required") used to leave an + # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: + # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form + # twice silently accumulated empty databases in their nav. + # + # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It + # does not test for `mode == "new"`; it asks "did the table this call's target resolved to + # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and + # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target + # rather than "any table that appeared", so a concurrent create in another request is never + # collateral. + before = set(ut_all(rt) or {}) + patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) + if terr: + return None, terr + minted = str((patch_cfg or {}).get("targetTable") or "") + minted = minted if minted and minted not in before else "" + if patch_cfg: + raw["config"] = {**(raw.get("config") or {}), **patch_cfg} + # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY + # has, instead of letting the pure validator mint a second empty one. See + # `discover_default_table`; a target the caller named is never rewritten. + raw = _apply_discover_default(rt, raw) + # W35-T35 (C8): `rt=` is the tenant wall for gated action kinds. Both doors pass it; a door + # that forgot would REFUSE the kind, not admit it (`TENANT_GATED_ACTIONS` is fail-closed). + defn, err = clean_definition(raw, None, username, notes=notes, rt=rt) + if err: + _unmint(rt, minted) + return None, err + # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations + # that already use them and NO NEW ONE CAN BE CREATED." + # + # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live + # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing + # the kind there would make both of them unsaveable — the ruling retires the door, not the + # automations behind it. Enforced on the server rather than left to the deleted wizard: a + # wall that holds only because the client stopped asking is a convention, not a wall. + if defn["kind"] in RETIRED_KINDS: + _unmint(rt, minted) + return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " + f"longer created. The ones already using it keep working. " + f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") + defn["id"] = _new_id(existing) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur[defn["id"]] = defn + return cur + + # `async` for the reason spelled out at `remove` and applied at `patch` — a create is the same + # blocking commit inside the same request, and the person is waiting on it in the same way. + # ⚠ NO PRESET GUARD HERE, deliberately: `patch` may skip the spawn because it can compare + # against a PREVIOUS definition, and a create has none — R10's columns must exist before a run, + # so this call stays unconditional. + _store_update(rt, _up, flush="async") + _presets_after_write(rt, defn, username) # R10: the columns exist before a run + if defn.get("trigger"): + _seed_event_state(rt, defn) + return defn, None + + +def patch(rt, auto_id, raw, username="", notes=None): + existing = all_definitions(rt) + prev = existing.get(str(auto_id)) + if prev is None: + return None, "no such automation" + merged = dict(prev) + # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a + # key missing here is accepted by the route, validated by `clean_definition`, and then + # DISCARDED — the client shows a saved flow that the store never received, and nothing goes + # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only + # ordering that never has a window where the bug exists. + for k in ("name", "config", "schedule", "kind", "trigger", "flow"): + if k in (raw or {}): + merged[k] = raw[k] + # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the + # Instagram trigger on an existing automation flips the kind, and the very next Save runs a + # config that has never carried a target through the pure validator, which mints the empty + # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. + merged = _apply_discover_default(rt, merged) + defn, err = clean_definition(merged, prev, username, notes=notes, rt=rt) + if err: + return None, err + defn["id"] = str(auto_id) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur[defn["id"]] = defn + return cur + + # ⛔ WHY THIS IS `async` — owner report 2026-08-12 (wave 30), MEASURED on the deployed build. + # Owner, verbatim: *"debug Automation module, its extremely slow, even renaming an Automation + # takes forever"*. A rename paid a BLOCKING HF commit inside the request, and `Store.update`'s + # sync branch holds `self._lock` across a strict DOWNLOAD **and** the upload — the same lock + # `Store.get` takes — so one rename stalls every OTHER reader of that tenant's store for its + # whole duration. That is the "module is slow" half a per-route fix never reaches. + # The reasoning, the read-your-writes contract and the `user_tables.add_row` precedent are + # written out in full at `remove` above (same wave, same owner report); this is that ruling + # applied to the save path rather than a second argument for it. + _store_update(rt, _up, flush="async") + # ⛔ THE PRESET SPAWN RUNS ONLY WHEN ITS OWN INPUTS MOVED — the other half of the same report. + # MEASURED as `nurilab-admin` on `4258a93`: **7,912 ms** for one rename against a **288 ms** + # warm `GET /automations` on the same connection, i.e. 27x the read this wave just made fast + # (T12/T13). A rename changes `name`; `_presets_after_write` reads `config`, `flow` and + # `trigger` and NOTHING else, then pays a full `user_tables` read (`ut_get`/`ut_ensure`; + # `rt.get` deep-copies by design, documented ceiling 35.8 MB / ~1.4 s) to re-derive columns + # that cannot have changed. ⭐ The generalisable half: T12/T13 took `ut_all` off the automation + # READ path this wave and every sibling WRITE path still carried it — a performance fix scoped + # to "the slow page" leaves the same call in every route beside it + # [[read-a-gates-predicate-for-what-it-excludes]]. + # ⛔ THE DANGEROUS DIRECTION IS SKIPPING, NEVER RUNNING, so the predicate is the INPUT SET of + # the function being skipped — not a hand-listed set of "cheap" edits, which is how a guard + # silently un-ships W25/R10 the next time the spawn learns to read a fourth key. When none of + # them moved the call is a no-op by construction (it re-derives from those same keys), and the + # run-time `ut_ensure` R10 deliberately kept is still the backstop if the TABLE drifted + # underneath — which is not a thing a rename should be repairing. Shaped after the trigger + # comparison directly below, which already compares `prev` against the cleaned definition. + # ⭐⭐ WAVE 31 · T30 — AND THE PREDICATE IS NOW THE SPAWN'S OWN ARGUMENT, which is the fix. + # The four-whole-objects test above was right in principle and inert in practice: `persist` + # sends a REBUILT `config`, so `prev["config"] != defn["config"]` on essentially every save and + # the guard fired every time — sparing only *rename*, the one edit the owner stopped + # complaining about after wave 30. `preset_inputs` is the canonical projection of exactly what + # `_spawn_presets` reads, and `_spawn_presets` is handed nothing else, so this comparison + # cannot go stale behind a future input the way a hand-listed key set would. The full argument, + # including why this is STRONGER than what it replaces rather than a relaxation, is in + # `preset_inputs`' own docstring — read that before widening or narrowing this line. + if preset_inputs(prev) != preset_inputs(defn): + _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS + # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching + # DISARMED — enabling never fires for records already in the state. + if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): + _seed_event_state(rt, defn) + return defn, None + + +def _presets_after_write(rt, defn, username): + """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run. + + Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset + columns immediately, "so R2b's list is real and the database is visible before anything is + spent". That last clause is the point — a person can look at the database, see the sixteen + columns, and decide whether to arm the schedule, instead of paying a vendor to find out what + they are agreeing to. + + ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs + nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first + run of a saved automation finds the table already correct and spends no commit. + + ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the + search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the + preset set, and spawning a subset here would mean the columns changed between Save and the + first run — two different answers to "what does this database look like", which is the exact + disagreement R10 exists to remove. + + ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure` + returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an + automation because the workspace is full would be a worse answer than saving one whose table + arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the + honest failure still has exactly one home. + """ + _spawn_presets(rt, preset_inputs(defn), username) + + +#: The enrich actions whose OWN config decides which preset columns and child databases a save +#: spawns, and the exact keys read off each. DECLARED here rather than spelled inside +#: `preset_inputs`, so `_spawn_presets` and the save-path guard cannot drift apart by one key. +PRESET_ACTION_INPUTS = { + "enrich_instagram": ("profileField",), + "enrich_tiktok": ("profileField", "postMetrics", "commentMetrics"), +} + + +def preset_inputs(defn): + """⭐⭐ WAVE 31 · T30 — EXACTLY the values `_spawn_presets` consumes off a definition, canonical. + + ⛔ THIS IS THE WHOLE POINT OF THE TICKET, so it is worth being precise about what changed and + why it is SAFER than what it replaces, not weaker. Owner item 7, verbatim: *"It still takes + forever to change the Config for automation as well. It just say Saving... and takes a long time + for me to change options and configs etc."* + + Wave 30 guarded the spawn on `any(prev[k] != defn[k] for k in ("config","flow","trigger", + "kind"))` and reasoned — correctly — that *"the predicate is the INPUT SET of the function being + skipped, not a hand-listed set of cheap edits"*. The flaw was not the reasoning; it was that the + input set was stated as four WHOLE OBJECTS while the spawn reads a handful of leaves out of + them. `AutomationDetail.tsx::persist` sends `config: buildConfig()` — a value RECONSTRUCTED from + React state, not the stored object — so the comparison is against something rebuilt on every + save and the guard never fires on the one edit the owner is complaining about. W30's fix spared + *rename* (the client omits `flow`/`trigger` entirely, and `kind`/`name` are unchanged), which is + exactly the edit the owner stopped complaining about after wave 30. + + ⭐ WHY NARROWING IS SAFE HERE AND WAS NOT SAFE THEN: this projection is not a hand-listed + allow-list that a future wave can silently outgrow. `_presets_after_write` hands this dict to + `_spawn_presets` and `_spawn_presets` **never receives `defn` at all** — so it is structurally + incapable of reading a fifth key that this function does not carry. Teaching the spawn to read + something new REQUIRES adding it here, and the guard then follows for free. That is a stronger + guarantee than wave 30's, which held only as long as somebody remembered the comment. + + ⚠ CANONICAL, because the input is a rebuild: each leaf is normalised exactly the way the spawn + itself normalises it (`str(...).strip()` for a field name, `bool(...)` for a switch). Without + that, `postMetrics: false` and a missing `postMetrics` would compare unequal while the spawn + treats them identically — a guard that fires on a difference its consumer cannot see is the + same defect in a smaller font. + + ⚠ IT DELIBERATELY COVERS THE DISCOVERY BRANCH **AND** THE ENRICH BRANCH IN ONE PROJECTION, even + though today's control flow returns before the second can run for a discovery automation. That + early return is D-178 (W31-T34) and it is going away; a projection built around today's branch + would silently narrow the guard the moment it does. + """ + defn = defn or {} + cfg = defn.get("config") or {} + actions = ((defn.get("flow") or {}).get("actions") or []) + return { + "target": str(_flow_table(defn) or cfg.get("targetTable") or ""), + "label": str(cfg.get("targetLabel") or ""), + "flowId": str(defn.get("id") or ""), + "discoveryKind": DISCOVERY_TRIGGER_KIND.get( + (defn.get("trigger") or {}).get("key") or ""), + "actions": { + kind: [{k: (_norm_switch(a, k) if k != "profileField" + else str((a.get("config") or {}).get(k) or "").strip()) + for k in keys} + for a in _actions_of_kind(actions, kind)] + for kind, keys in PRESET_ACTION_INPUTS.items() + }, + } + + +def _norm_switch(action, key): + """A capture switch as the spawn reads it — `bool`, so absent and False are ONE value.""" + return bool((action.get("config") or {}).get(key)) + + +def _spawn_presets(rt, inputs, username): + """The spawn itself. Takes `preset_inputs(defn)` — never the definition — see that docstring. + + ⭐⭐ WAVE 31 · T30, THE SECOND HALF: **ONE `user_tables` read for the whole pass, not five.** + Every `ut_ensure` below used to take its own copy of the tenant document through + `ut_get` → `ut_all` → `rt.get`, which deep-copies unconditionally (ceiling 35.8 MB / ~1.4 s), and + the TikTok arm reaches FOUR of them plus the `ut_get` above — measured **7,912 ms** on + `4258a93`. Lending one snapshot is the same fix W30-T13 made on the automation READ path + (`routes_automation.py::_LentTables`); this is deliberately NOT a third copy of that class, but + the `tables=` parameter `retire_automation_stage_fields` in this very module already + established, because here every consumer is a function we own and pass to directly. + """ + target = inputs["target"] + if not target: + return + tag = inputs["flowId"] + ig_actions = inputs["actions"]["enrich_instagram"] + tt_actions = inputs["actions"]["enrich_tiktok"] + try: + # ⭐ THE ONE READ. Everything below answers out of this snapshot. + tables = ut_all(rt) + # ⭐⭐ WAVE 30 · T06 — BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was + # R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure` + # at exactly ONE site — inside `run_discover_tiktok` — so the database did not exist until + # money had already been spent finding out what was in it. That is the precise pre-R10 + # behaviour `verify_automation`'s NC39 forbids on the Instagram side. + # ⛔⛔ THE DISCRIMINATOR IS THE **TRIGGER**, NOT THE KIND, AND THE GATE PAID FOR THAT + # SENTENCE. Widening this to `kind in DISCOVERY_KINDS` looks equivalent — law 1 makes a + # discovery trigger choose its kind — but the implication only runs ONE WAY. A definition + # may carry `kind: "discover_instagram"` with **no trigger at all**: law 1 sets the kind + # from the trigger and never the reverse, so any direct API create can do it, and one of + # this repo's own fixtures does. MEASURED: the kind test spawned a preset database under a + # dry-run fixture whose check asserts that no table exists — a red on correct-looking code, + # caught only because that check happened to exist. + # ⇒ R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own + # answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1. + # ⚠ TRACKED because it is the ONE key several `ut_ensure` calls in this pass can share: the + # discovery arm and both enrich arms all ensure `target`. A later one must not answer out of + # a snapshot an earlier one invalidated — see the `tables=` arguments below. + wrote_target = False + _dkind = inputs["discoveryKind"] + if _dkind: + _, _, _spawn_label, _spawn_fields = discovery_facts(_dkind) + ut_ensure(rt, inputs["label"] or _spawn_label, _spawn_fields, username, + key=target, flow_tag=tag, lock_fields=True, tables=tables) + # ⭐⭐ WAVE 31 · T34 (D-178) — AND THE `return` THAT USED TO BE HERE IS GONE. + # + # ⛔ WHAT IT COST: a DISCOVERY automation could never spawn its child databases. The + # capture switches below are the ones that promise `ut_tt_posts` / `ut_tt_comments` / + # `ut_tt_post_snapshots` at SAVE, before any money is spent (W30-T10) — and only a + # `plain` flow ever reached them, because a discovery flow left this function three + # lines earlier. So the exact automation the owner would build for TikTok discovery — + # find profiles, then capture their posts — silently got the profile table and nothing + # else, and the toggles it showed were promises the save path never kept. + # ⚠ IT IS A TRAP, NOT TODAY'S ONLY SYMPTOM: nurilab's child tables are absent for a + # CONFIGURATION reason as well, so fixing this alone will not make them appear there. + # + # ⚠ THE SNAPSHOT IS NOW STALE FOR `target`, and the arms below resolve their profile + # binding out of that very table (`bound` reads its `fields`). One re-read, paid only on + # a save that actually reached the discovery spawn — correctness before the read count, + # and it is one read against the 41 this function used to take. `wrote_target` stays + # False precisely BECAUSE we refreshed: it tracks staleness, not "did somebody write". + tables = ut_all(rt) + enrich_actions = ig_actions + table = tables.get(target) or {} + named = next((a["profileField"] for a in enrich_actions if a["profileField"]), "") + bound = next((str(f.get("key") or "") for f in table.get("fields") or [] + if isinstance(f.get("profile"), dict)), "") or named + if enrich_actions and bound: + ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username, + key=target, flow_tag=tag, lock_fields=True, tables=tables) + wrote_target = True + # ⭐ WAVE 30 · T08 — the TikTok half of W25/R10: the columns an `enrich_tiktok` step will + # fill appear when the automation is SAVED, not when money is first spent. Resolved + # through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an + # Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up + # uses — R10's rule is that the column set does not change between Save and the first run, + # and one shared resolver is what makes that structural instead of a convention. + if tt_actions: + tt_named = next((a["profileField"] for a in tt_actions if a["profileField"]), "") + tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT) + if tt_bound: + ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound), + username, key=target, flow_tag=tag, lock_fields=True, + tables=None if wrote_target else tables) + # ⭐ WAVE 30 · T10 — and the CHILD databases the step's own switches promise, at + # SAVE, before any money. R10's rule is "the columns a step will fill appear when + # you press Save"; a `postMetrics` toggle whose database only exists after a paid + # run is the same complaint one level up. + # ⛔ GATED ON THE SWITCHES, never spawned unconditionally: a database that exists + # because somebody looked at a toggle, and then never fills, is the "SECOND, empty + # database" the discovery default was rewritten to stop producing. + for _flag, _child in (("postMetrics", TT_POSTS_TABLE), + ("postMetrics", TT_POST_SNAPSHOTS_TABLE), + ("commentMetrics", TT_COMMENTS_TABLE)): + if any(a[_flag] for a in tt_actions): + # R9 (W31-T32): the child arrives LOCKED, from its own declaration. + ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username, + key=_child, flow_tag=tag, lock_fields=True, tables=tables, + record_mode=tt_record_mode(_child)) + except Exception as e: # noqa: BLE001 + # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so + # raising here would answer 500 for an automation that IS stored — the caller would retry + # and create a second one. The columns then arrive on the first run, which is precisely + # the behaviour this function is an improvement on rather than a replacement for. + print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}") + + +def remove(rt, auto_id): + aid = str(auto_id) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur.pop(aid, None) + return cur + # ⛔ WHY THIS IS `async` AND NOT `sync` (owner report 2026-08-12, wave 30, MEASURED). + # Owner, verbatim: *"it takes forever to delete an automation"*. A delete was paying up to + # TWO BLOCKING UPLOADS inside the request — this one, plus the whole-document rewrite below, + # which fires for every DISCOVERY automation because those always tag the columns they + # spawned (deleting one marked all 30 columns of `ut_tt_profile` disabled). Measured 977 ms + # for the CHEAPEST case (a `plain` flow, no tagged columns) against a 757 ms `GET /automations` + # baseline on the same connection; the discovery case is strictly worse by a full-document + # read plus a full-document commit. + # + # `flush="async"` is not a weakening: `Store.update`'s async branch applies the mutation to + # the in-process cache, marks it owned+dirty and schedules a COALESCING worker, and the + # class's read-your-writes contract means the very next `GET /automations` already sees the + # deletion. This is the same trade `user_tables.add_row` makes for ROW creation — strictly + # more valuable data than an automation definition — so a delete is not the place to be + # stricter than a row insert. [[sync-write-eats-pending-async-write]] is the hazard in the + # OTHER direction (a sync writer discarding a pending async write) and `Store.update`'s dirty + # branch already handles it. + _store_update(rt, _up, flush="async") + # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned + # silently: the column keeps its values and its config, and says why it stopped. Scanned + # first so a delete with no tagged fields costs no store commit. + # ⚠ HONEST RESIDUAL, stated rather than smoothed: this scan still costs ONE full read of the + # tenant's `user_tables` document on EVERY delete (`rt.get` deep-copies by design), and the + # update below reads it a second time. Removing that needs a tagged-field index, which is a + # feature and not a wave-tail edit — booked rather than improvised. What it no longer costs + # is the part the owner could feel: the blocking network commits. + # ⭐ WAVE 31 · T39(a) — D-177(a): ONE WALK, AND THE PLAN CARRIES WHAT IT FOUND. + # This used to scan the whole document to answer a boolean (`hit`), then walk the WHOLE + # document again inside `_disable` to re-derive the same matches. The scan now records the + # `(table, field)` pairs it found and the mutation applies exactly those — the shape + # `bind_unbound_fields` in this module already uses, so a delete costs one walk instead of two + # and the write touches only the fields it named. + # ⭐⭐ WAVE 33 (D-179) — THE SCAN NO LONGER READS THE ROWS, and that is where the cost was. + # + # This walk asks ONE question — "which fields did this flow tag?" — and it has never looked at + # a row to answer it. It was nevertheless taking `ut_all`, a whole deep copy of the tenant's + # `user_tables` document, of which **99.9% of the bytes are `rows` nothing here reads** + # (measured on tenant #0: 28.6 MB, 81,330 rows, 703 ms warm). A's C5 projection makes that + # ~0.1% of the bytes. + # ⛔ HANDED TO A READ AND NOTHING ELSE. `lend_defs` serves `_Projected` values, so reaching + # `defn['rows']` raises a `KeyError` NAMING the projection — by design — and `plan` carries + # only `(table_key, field_key)` STRINGS out of this scope. The mutation below takes its own + # strict document from `rt.update` and never sees a projected value, which is C5's second + # clause: a projected snapshot is never handed to a post-write read-back. + # ⚠ HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): D-179's exit condition + # asks for ZERO full-document reads and this is ONE — `rt.update` below takes a strict read + # that belongs to the WRITE and cannot be removed from here. What is gone is the expensive one. + # ⚠ AND THE FALLBACK IS DELIBERATE: `all_defs` degrades to the whole read on any failure, so a + # store that cannot project still deletes correctly, only slower. + # + # ⭐⭐ WAVE 35 · T34 — THE RESIDUAL IS NOW MEASURED RATHER THAN ASSERTED, and the count is + # better than the paragraph above claims. Whole `user_tables` reads per delete, against a + # runtime that has production's `get_projection`: + # TAGGED (a discovery flow that spawned columns) → 1 whole + 1 projected + # UNTAGGED (a `plain` flow) → 0 whole + 1 projected + # The untagged case pays NONE because `if not plan: return` fires before the write. The + # tagged case's ONE is `rt.update`'s own strict read, and it is a FLOOR, not a choice: + # `core/store.py::get_projection` states that a write must always read whole, because a + # read-modify-write handed a rows-less document would upload the tenant with its rows deleted. + # ⛔ Reaching ZERO needs D-179's `{flowId: [(table_key, field_key)]}` index, written where the + # presets are spawned — a feature, not an edit to this function. Nothing here can do better. + # ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double + # had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the + # PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts. + try: + import core.user_tables as _ut_defs # noqa: PLC0415 + _scan = dict(_ut_defs.all_defs(rt) or {}) + except Exception: # noqa: BLE001 + _scan = ut_all(rt) + plan = [(tk, str(f.get("key") or "")) + for tk, t in _scan.items() + for f in ((t or {}).get("fields") or []) + if str((f.get("automation") or {}).get("flowId") or "") == aid] + if not plan: + return + + def _disable(cur): + cur = cur if isinstance(cur, dict) else {} + note = f"its automation was deleted {_stamp()}" + for tk, fk in plan: + for f in ((cur.get(tk) or {}).get("fields") or []): + a = f.get("automation") + if f.get("key") == fk and isinstance(a, dict): + a["disabled"] = True + a["statusNote"] = note + return cur + # The expensive half of the owner's complaint: a whole-document rewrite, committed while the + # person waits. Async for the reason given above — the disable is visible to the next read + # immediately; only the upload is deferred, and it coalesces with any other pending write to + # the same key instead of racing it. + rt.update(UT_STORE_KEY, _disable, flush="async") + + +# --------------------------------------------------------------------------------------------- +# RUN STATE — process memory only (see the module header for why this may never be persisted) +# --------------------------------------------------------------------------------------------- + +_RUN_LOCK = threading.RLock() +_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} + + +def running(tenant, auto_id): + with _RUN_LOCK: + return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None + + +def _claim(tenant, auto_id): + with _RUN_LOCK: + if (tenant, str(auto_id)) in _RUNNING: + return False + _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} + return True + + +def _step(tenant, auto_id, text): + with _RUN_LOCK: + cur = _RUNNING.get((tenant, str(auto_id))) + if cur is not None: + cur["step"] = text + + +def _no_step(_text): + """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner + directly). A runner must never REQUIRE the slot — the slot is process state, and the runner + is the part a test is allowed to call on its own.""" + + +def _release(tenant, auto_id): + with _RUN_LOCK: + _RUNNING.pop((tenant, str(auto_id)), None) + # A completed/manual/event run may have changed rows that feed a metric, link or source + # rollup. Mark the derived lane dirty; the next durable tick will refresh it once. Merely + # waking the scheduler does not mark anything dirty and therefore does not poll Neon. + _mark_derived_dirty(tenant) + + +def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None): + """THE ONE store write a run performs on the `automations` bucket. + + `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here + because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to + invent a node status when this map is absent (every run stored before W19-C has no `steps`, + and painting those nodes green from the run's overall state would be a fabrication). + + ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the + comprehension below drops everything else — so the per-record sentence a vendor gave us had + literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the + product could say. MEASURED on nurilab: the same record blocked on three separate runs and + the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by + calling the vendors by hand. The note is the most valuable thing a run produces — a dead + handle, a vendor hiccup and an exhausted key are three different actions — and it was the one + thing thrown away. + ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it + did not ask for, and W25/R4 already retired writing status STRINGS into people's grids. + """ + entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), + "counts": {k: int(v) for k, v in (counts or {}).items() + if isinstance(v, (int, float))}, + "steps": {str(k): str(v) for k, v in (steps or {}).items()}, + # Bounded on both axes: a 100-profile run must not put 100 sentences into a store + # entry that is kept 20 deep per automation. + "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25], + "affected": list(affected or [])[:200]} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is None: + return cur + d["status"] = {"state": state, "lastRunAt": entry["ts"], + "lastSummary": entry["summary"]} + d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] + # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with + # the reason on it — a dead credential must not burn quota (or a paid vendor's records) + # 96 times a day while a green toggle sits over a buried error log. Errors only: + # `partial` is the honest-progress state and pausing on it would punish honesty. + runs = d.get("runs") or [] + if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( + not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): + sch = d.get("schedule") + if isinstance(sch, dict) and sch.get("enabled"): + sch["enabled"] = False + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " + f"failures. Fix the cause, then re-enable: " + f"{entry['summary'][:120]}") + return cur + + _store_update(rt, _up, flush="sync") + _notify_outcome(rt, auto_id, state, entry) + return entry + + +def _notify_outcome(rt, auto_id, state, entry): + """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7). + + Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is + written, so it is the only place this can go without a second definition of "what happened". + + ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED. + 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the + MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a + background thread with no session, so an omitted `st` files nurilab's automation failures + in Royal Imports' inbox — D-16's class, silently, forever. + 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler + has no session at all, so the alternative is not "the wrong person" but "nobody", and + `notify` returns None on a blank owner — the notification would simply never exist, on + exactly the runs nobody was watching. + ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only + view-alert CREATION; `notify()` writes straight into the notifications bucket and + `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell. + ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that + SUCCEEDED into one that raised — the run entry is already committed one line above, and the + bell is a courtesy on top of it. + """ + try: + defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {} + owner = str(defn.get("createdBy") or "").strip() + if not owner: + return + from core import alerts as _alerts + _alerts.notify( + owner=owner, + # ⭐ WAVE 34 · R12 — the fallback name a notification wears when the automation has + # none. "Agent", singular: this names ONE of them, not the module. + label=_s(defn.get("name") or "Agent", 80), + topic="automation", + key=str(auto_id), + detail=f"{state}: {entry.get('summary') or ''}"[:300], + st=rt) + except Exception: # noqa: BLE001 + pass + + +# --------------------------------------------------------------------------------------------- +# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN +# --------------------------------------------------------------------------------------------- +# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and +# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) +# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: +# +# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, +# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the +# supplier takes the scraping risk; it is never an Instagram credential and this module must +# never be given one. +# +# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). +# +# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress +# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been +# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and +# the cell says which. A pull that quietly wrote zero posts and reported success would be worse +# than no automation at all: the table would look maintained and be empty. +# +# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, +# so a run history answers "what still works anonymously?" from data rather than from memory. + +PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) + +#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's +#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number +#: is a NOW value, so the series starts the day capture starts and a field we do not capture +#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows +#: has exactly the same cost as delaying capture — for that column — and is justified by exactly +#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to +#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. +#: Every returned provider field is retained in Source data. Promoted columns make commonly used +#: values filterable; the raw source document prevents a new or uncommon field from being lost. +#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose +#: almost none of them, which is what the `source` column is for. +SNAPSHOT_FIELDS = [ + field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), + field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"), + field_def("following", "Following", "int"), + field_def("posts_count", "Post count", "int"), + field_def("full_name", "Name"), field_def("bio", "Bio"), + field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"), + field_def("approx", "Counts are approximate", "checkbox"), + # The link in bio — the paid rung's field, and commercially the most useful single string an + # influencer row can carry (it is the shop/affiliate destination). + field_def("external_url", "Link in bio", "url"), + # --- the rest of the vendor's Profiles schema. + field_def("ig_id", "Instagram id"), + field_def("profile_url", "Profile", "url"), + field_def("avg_engagement", "Avg engagement", "pct"), + field_def("category", "Category"), + field_def("business_category", "Business category"), + field_def("is_business", "Business account", "checkbox"), + field_def("is_professional", "Professional account", "checkbox"), + field_def("is_private", "Private", "checkbox"), + field_def("highlights_count", "Story highlight count", "int"), + field_def("bio_hashtags", "Bio hashtags"), + field_def("pronouns", "Pronouns"), + # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot + # row maps the WHOLE schema by design, so these belong here the moment the map reads them; + # leaving them out would be the "captured but unrecorded" half of the same loss the note at + # the top of this list is about. + field_def("profile_name", "Profile name"), + field_def("is_joined_recently", "Joined recently", "checkbox"), + field_def("has_channel", "Has channel", "checkbox"), + field_def("partner_id", "Partner id"), + field_def("external_url_title", "Link title"), + field_def("fbid", "Facebook id"), + field_def("related_accounts", "Related accounts"), + field_def("country_code", "Country"), + field_def("source_payload", "Source data", "json"), +] +POST_FIELDS = [ + field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), + field_def("posted_at", "Posted at", "date"), + field_def("type", "Type", "select", options=["image", "video", "carousel"]), + field_def("caption", "Caption"), field_def("url", "URL", "url"), + # A tagged place is useful content context and can be filtered as ordinary text. It is never + # presented as the creator's location: a creator can tag a holiday, venue or brand location. + field_def("tagged_location", "Tagged location"), + # The per-field map below gives operational fields first; this locked JSON document retains + # every other value Bright Data supplied, so a vendor schema addition is preserved + # instead of being silently discarded while the canonical field model catches up. + field_def("source_payload", "Source data", "json"), + # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous + # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). + field_def("paid_partnership", "Paid partnership", "checkbox"), + field_def("partner", "Partner brand"), + field_def("hashtags", "Hashtags"), + field_def("alt_text", "Alt text"), + # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT + # ONE HOP. + # + # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's + # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a + # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated + # transitively). So the post row carries its own LATEST value and the rollup reads it + # directly. + # + # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile + # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in + # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that + # was just appended — they are a projection of that store, never a second copy of it, and + # deleting them would cost a convenience rather than a measurement. + # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is + # off by default (it buys one vendor record per post), so on most pulls this stays empty, and + # empty means "not read", which is what makes an honest average possible at all. A 0 here + # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or + # COMMENT count is a real measurement and must survive. + # + # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08, + # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought). + # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct + # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in + # the very same rows. Mapping it here is what put one number on many unrelated posts. We + # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers; + # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays + # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained + # whole, so if the vendor ever populates it per-reel the history is re-derivable. + # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30 + # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the + # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named + # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10 + # (folded into a single "Views"), not because we are reading the wrong key. + # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED. + # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the + # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` — + # measured against a browser-read ground truth to the digit. Same column name, same type, same + # rollup; the engine underneath is swappable and the preset database never moved. That is the + # owner's schema ruling working exactly as intended. + # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung. + field_def("views", "Views", "int"), + field_def("plays", "Plays", "int"), + field_def("likes", "Likes", "int"), + field_def("comments", "Comments", "int"), + # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data + # further"*). Two fields the primary source does not return AT ALL, already paid for inside + # the same engagement response — so capturing them costs nothing extra. + # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on + # every write door, so a normaliser that emits `video_duration` without this line writes + # nothing and reports success — the wave-28 defect that swallowed nine preset cells. + field_def("video_duration", "Video length (s)", "int"), + field_def("comments_disabled", "Comments off", "checkbox"), + field_def("measured_at", "Engagement read at", "date"), + # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED": + # a post reaches its own comment rows the same way a profile reaches its posts. Derived from + # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance. + # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state + # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when + # `postMetrics` is off. + field_def("comments_link", "Comment rows", "link", + description="Comment records linked to this post.", + link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), + field_def("post_snapshots_link", "Measurement rows", "link", + description="Engagement measurements linked to this post.", + link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), + field_def("measurements_captured", "Measurements captured", "rollup", + rollup={"link": "post_snapshots_link", "fn": "countall"}), +] + +#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE. +#: +#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to +#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default +#: is the capture. +#: +#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED. +#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE: +#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of +#: `instagram-capture.md`). They are the single most expensive thing this product can buy. +#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has +#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment +#: thread ingests identifiable people who never entered anybody's influencer list. +#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code +#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments` +#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on +#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is +#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md` +#: is written against. +#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as +#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs +#: nothing; the spend and the third-party ingest are a click somebody makes knowingly. +#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because +#: they are part of a record already paid for. The separate full Comments dataset remains OFF +#: unless `commentMetrics` is explicitly enabled on the enrichment action. +COMMENT_FIELDS = [ + field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"), + field_def("influencer_key", "Influencer"), + # ⭐⭐ OWNER RULING 2026-08-12 — the same one that put `text` on the TikTok comment schema, and + # it lands on BOTH networks in the same change on purpose: the two comment tables are read side + # by side, and one carrying the content while the other does not is the divergence this repo + # keeps paying for. ⛔ TEXT ONLY — `comment_user` / `comment_user_url` are vendor-FLAGGED PII + # and stay in `source_payload`, uncolumned. + field_def("text", "Comment"), + field_def("commented_at", "Commented at", "date"), + field_def("likes", "Likes", "int"), + field_def("replies", "Replies", "int"), + field_def("source_payload", "Source data", "json"), + field_def("post_link", "Post", "link", + description="Post record linked to this comment.", + link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), + # Author/text and every provider-specific value stay intact in Source data. The promoted + # columns intentionally keep the grid concise while Likes and Replies make comment engagement + # directly filterable and rollup-ready. +] +POST_SNAPSHOT_FIELDS = [ + field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), + field_def("influencer_key", "Influencer"), + field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"), + field_def("comments", "Comments", "int"), + # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits + # before any rollup argument is made. + # + # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with + # only the first can describe a measurement but not its AGE, so the single most useful thing + # this series can compute — views at N days old, i.e. velocity — is not expressible over it at + # all. `type` is the same shape one column over: "reels only" is the default lens on this data + # and without it every question has to go back through `ut_ig_posts` to ask what kind of post + # this was. + # + # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row + # from the same vendor record), so this costs one dict key each and no extra call. + # + # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24 + # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the + # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied + # from it deliberately rather than shared, because these are two different tables' columns that + # happen to agree today, and `verify_automation` asserts the agreement. + # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO + # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`, + # and `ut_ig_posts` already performs the first of them. See the verdict in + # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for + # moving the window here. + field_def("posted_at", "Posted at", "date"), + field_def("type", "Type", "select", options=["image", "video", "carousel"]), + # Plays move independently of likes on video, so it is its own series rather than a thing to + # derive. Paid rung only; blank means not read. + # The series regains Views alongside the latest-value projection on the Post row — one store + # for one series (R3) is untouched; this IS that store. + field_def("views", "Views", "int"), + field_def("plays", "Plays", "int"), + field_def("source_payload", "Source data", "json"), + field_def("post_link", "Post", "link", + description="Post record linked to this measurement.", + link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), +] + +#: ⭐⭐ WAVE 32 · T41 — THE INSTAGRAM TWIN OF `TT_TABLE_FIELDS`, WHICH DID NOT EXIST. +#: +#: `TT_TABLE_FIELDS`' own note said so and named the consequence: *"The Instagram side has no +#: equivalent map, which is exactly why its table names are scattered across the module."* It is a +#: map now, for a concrete reason rather than symmetry: A's boot-time delivery sweep (`W32-T07`) has +#: to call `ut_ensure` for every locked child of BOTH platforms, and a sweep that can ask TikTok for +#: its table set and must hand-type Instagram's is one platform away from the drift this pair of +#: maps exists to stop. +#: +#: ⛔ THESE FOUR LISTS ARE THE *CANONICAL* DECLARATION — the per-tenant BACKLINK fields +#: `ensure_ig_graph` appends are NOT here, and must not be. A backlink is derived per profile +#: database (`_profile_backlink_field`), so folding it into a module constant would make one +#: tenant's relation a global fact. `ensure_ig_graph` reads this map and adds its own backlinks on +#: top, which is why that function still owns the graph write and this only owns the schema. +IG_TABLE_FIELDS = { + IG_SNAPSHOTS_TABLE: SNAPSHOT_FIELDS, + IG_POSTS_TABLE: POST_FIELDS, + IG_POST_SNAPSHOTS_TABLE: POST_SNAPSHOT_FIELDS, + IG_COMMENTS_TABLE: COMMENT_FIELDS, +} +#: The labels those four wear in the database list. ⚠ LIFTED VERBATIM from `ensure_ig_graph`'s own +#: `graph` literal, which is now derived from this map — a renamed label here renames the table +#: everywhere rather than leaving two spellings of one database. +IG_TABLE_LABELS = { + IG_SNAPSHOTS_TABLE: "IG snapshots", + IG_POSTS_TABLE: "IG posts", + IG_POST_SNAPSHOTS_TABLE: "IG post snapshots", + IG_COMMENTS_TABLE: "IG comments", +} + + +# --------------------------------------------------------------------------------------------- +# ⭐⭐ TIKTOK — THE FIVE `ut_tt_*` SCHEMAS (wave 29 · item 7 · D-9 · rulings R1 + R2) +# --------------------------------------------------------------------------------------------- +# ⛔ EVERY VENDOR FIELD NAME BELOW IS FROM ONE PROBED SOURCE — `waves/wave29/proto/tiktok-schema.md`, +# 40 profile / 43 post / 17 comment fields read live from `GET /datasets/{id}/metadata` for $0.00, +# each carrying the vendor's own type, description and `pii` flag. NOTHING HERE IS GUESSED, and the +# names live in `connectors_tt.py` (the map), not here (the schema). This file declares what a +# COLUMN is; the connector declares what the wire calls it. +# +# ⭐ THE SAME LAW AS THE INSTAGRAM LISTS ABOVE: map the schema, not just what was populated on the +# probe. HISTORY IS UNBUYABLE — no vendor sells "followers on 1 January" — so a field we do not +# capture today is permanently lost for today, and an occasionally-empty column costs a blank cell +# while a missing one costs the months before somebody notices. +# +# ⚠ TWO DELIBERATE DIVERGENCES FROM THE INSTAGRAM SCHEMA, both licensed by R2 ("the two schemas may +# diverge where the vendors do") and both recorded so neither reads as an omission: +# +# 1. NO `plays` COLUMN. TikTok returns ONE number, `play_count`, and it is the count TikTok +# itself displays under a video — i.e. our `views`. Instagram has two columns because Meta +# once had two metrics (`plays` retired 2025-04-10). Writing one vendor number into two of our +# columns would manufacture a second measurement that a rollup could average or double-count, +# and blank-vs-zero discipline says an unmeasured column must be blank, not a copy. +# ⇒ `play_count` -> `views`, and `plays` does not exist on this family. +# 2. `tt_id`, NOT `ig_id`. The probe doc flags this: our profile key is literally NAMED `ig_id`. +# A TikTok row carrying a column labelled "Instagram id" is a header that lies, and this is a +# NEW table family with no stored rows to migrate — so the honest name costs nothing. +# +# ⚠ THE COLUMNS THAT STAY BLANK ARE NAMED RATHER THAN DROPPED SILENTLY. TikTok has no equivalent of +# `category`, `business_category`, `is_professional`, `highlights_count`, `bio_hashtags`, +# `pronouns`, `profile_name`, `is_joined_recently`, `has_channel`, `partner_id`, +# `external_url_title`, `fbid` or `related_accounts` (profile), nor of `alt_text`, +# `comments_disabled`, `paid_partnership` or `partner` (post) — `commerce_info` is a LOCATION, not +# a paid-partnership flag, and TikTok's `comment_setting` is profile-level rather than per-post. +# Those columns are ABSENT here rather than declared and permanently blank — a column nothing can +# ever write is a promise the grid keeps making that the vendor cannot keep. + +#: ⭐ WAVE 29 (contract C2) — the value the `profile` FLAG carries on a TikTok handle column: the +#: twin of `PROFILE_SOURCE_IG`, which is declared further down beside the `PLATFORM_*` vocabulary +#: with the note on why those two families of string are NOT the same thing. It sits here rather +#: than there because the lists below are evaluated at IMPORT and would raise a NameError otherwise. +#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT, AND THE OTHER HALF HAS LANDED (verified 2026-08-12, +#: W30-T08): `platform/core/user_tables.py:PROFILE_SOURCES` now reads `('instagram', 'tiktok')`. +#: Before it did, `_clean_profile` REFUSED this flag and the handle column of a `ut_tt_profile` +#: spawn was dropped — fail-closed and loud by design, because the alternative is a profile +#: database whose profile column silently is not one. Left written down rather than deleted: the +#: refusal is what a TikTok binding looks like on any deployment where that line is missing. +PROFILE_SOURCE_TT = "tiktok" + +#: The profile row: LATEST values plus `enriched_at`. The SERIES lives in `ut_tt_snapshots` — +#: R3's "one store for one series" carries over unchanged, because the reason for it does (no +#: vendor sells history, so the append table is the only history there will ever be). +TT_PROFILE_FIELDS = [ + # `platform` FIRST, exactly as the Instagram preset set has it — and on all five tables here, + # not just this one. The IG family carries it on the profile table alone, which is why a TikTok + # POST could never have lived beside an Instagram post; declaring it everywhere is what makes a + # future cross-platform union view a UNION rather than a guess. + tt_field_def("platform", "Platform"), + tt_field_def("handle", "Handle", pinned=True, + profile={"source": PROFILE_SOURCE_TT}), + tt_field_def("full_name", "Name"), + tt_field_def("tt_id", "TikTok id"), + tt_field_def("profile_url", "Profile", "url"), + tt_field_def("bio", "Bio"), + tt_field_def("external_url", "Link in bio", "url"), + tt_field_def("verified", "Verified", "checkbox"), + tt_field_def("is_private", "Private", "checkbox"), + tt_field_def("is_business", "Business account", "checkbox"), + tt_field_def("followers", "Followers", "int"), + tt_field_def("following", "Following", "int"), + tt_field_def("posts_count", "Video count", "int"), + # TikTok gives THREE engagement rates where Instagram gives one. All three are stored ×100 for + # the same reason `avg_engagement` is on the IG side (C1-a): the vendor sends a 0–1 fraction and + # our `pct` renderer appends the sign to the stored number, so the raw fraction would print a + # 6.6% creator as `0.0%`. + tt_field_def("avg_engagement", "Avg engagement", "pct"), + tt_field_def("like_engagement", "Like engagement", "pct"), + tt_field_def("comment_engagement", "Comment engagement", "pct"), + # Total likes RECEIVED across the account's videos — a TikTok-only number with no Instagram + # equivalent, and one of the few profile-level engagement facts a vendor gives away. + tt_field_def("likes_received", "Likes received", "int"), + tt_field_def("country_code", "Country"), + tt_field_def("region", "Region"), + tt_field_def("predicted_lang", "Language"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP. The vendor's `create_time` on a profile is when the + # ACCOUNT was created; it is emphatically not "when `followers` was true". TikTok carries no + # measurement timestamp at all, exactly like Instagram — which is why the append law below + # (`@`) is the only thing that can date a number. + tt_field_def("account_created_at", "Account created", "date"), + tt_field_def("first_found", "First found", "date"), + tt_field_def("last_found", "Last found", "date"), + tt_field_def("found_count", "Times found", "int"), + tt_field_def("created_by", "Found by"), + tt_field_def("enriched_at", "Enriched at", "date"), + tt_field_def("source", "Read via"), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("profile_snapshots_link", "Measurement rows", "link", + description="Profile measurements linked to this account.", + link={"table": TT_SNAPSHOTS_TABLE, "on": "influencer_key", "from": "handle"}), + tt_field_def("posts_link", "Post rows", "link", + description="Post records linked to this account.", + link={"table": TT_POSTS_TABLE, "on": "influencer_key", "from": "handle"}), +] + +#: The profile SERIES. One row per profile per pull, keyed `@` — the append law +#: from `instagram-capture.md` §3, unchanged, because its cause is unchanged: the vendor stamps +#: nothing, so the only honest date a number can carry is the moment WE read it. +TT_SNAPSHOT_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("snapshot_key", "Snapshot"), + tt_field_def("influencer_key", "Influencer"), + tt_field_def("pulled_at", "Pulled at", "date"), + tt_field_def("followers", "Followers", "int"), + tt_field_def("following", "Following", "int"), + tt_field_def("posts_count", "Video count", "int"), + tt_field_def("likes_received", "Likes received", "int"), + tt_field_def("full_name", "Name"), + tt_field_def("bio", "Bio"), + tt_field_def("verified", "Verified", "checkbox"), + tt_field_def("is_private", "Private", "checkbox"), + tt_field_def("is_business", "Business account", "checkbox"), + tt_field_def("avg_engagement", "Avg engagement", "pct"), + tt_field_def("like_engagement", "Like engagement", "pct"), + tt_field_def("comment_engagement", "Comment engagement", "pct"), + tt_field_def("external_url", "Link in bio", "url"), + tt_field_def("tt_id", "TikTok id"), + tt_field_def("profile_url", "Profile", "url"), + tt_field_def("country_code", "Country"), + tt_field_def("region", "Region"), + tt_field_def("predicted_lang", "Language"), + tt_field_def("account_created_at", "Account created", "date"), + tt_field_def("source", "Read via"), + tt_field_def("approx", "Counts are approximate", "checkbox", + description="Checked when the counts on this row are rounded, not exact."), + tt_field_def("source_payload", "Source data", "json"), +] + +#: One row per TikTok post. Identity is `shortcode`, and on TikTok that is a 19-digit numeric id — +#: ✅ the SAME shape as the Comments dataset's `post_id`, so the comments→posts link joins on +#: equality with NO normaliser (measured in the probe doc; Instagram needed one). +TT_POST_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("shortcode", "Post id"), + tt_field_def("influencer_key", "Influencer"), + tt_field_def("posted_at", "Posted at", "date"), + # ⛔ THE VENDOR'S VOCABULARY IS `"video"` / `"content"` AND OURS HAS NO `"content"`. Declaring + # the vendor's word would put an untranslated API token in front of a user; declaring an option + # the mapper can emit but the select does not list would fail `_clean_field`. So the OPTIONS + # stay this product's words and `connectors_tt.normalize_post` does the translation — the same + # posture every other vendor value here takes. `carousel` is listed because a TikTok photo post + # carrying more than one image IS one, and the mapper decides from `carousel_images` rather + # than from the type token, which cannot express it. + tt_field_def("type", "Type", "select", options=["image", "video", "carousel"]), + tt_field_def("caption", "Caption"), + tt_field_def("url", "URL", "url"), + tt_field_def("hashtags", "Hashtags"), + tt_field_def("tagged_location", "Commerce location"), + # ⛔ `views` AND NO `plays` — see the divergence note at the top of this section. One vendor + # number, one column. + tt_field_def("views", "Views", "int"), + tt_field_def("likes", "Likes", "int"), + tt_field_def("comments", "Comments", "int"), + tt_field_def("shares", "Shares", "int"), + tt_field_def("saves", "Saves", "int"), + tt_field_def("video_duration", "Video length (s)", "int"), + tt_field_def("measured_at", "Engagement read at", "date", + description="When the engagement numbers on this row were read."), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("comments_link", "Comment rows", "link", + description="Comment records linked to this post.", + link={"table": TT_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), + tt_field_def("post_snapshots_link", "Measurement rows", "link", + description="Engagement measurements linked to this post.", + link={"table": TT_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), + tt_field_def("measurements_captured", "Measurements captured", "rollup", + description="How many measurements this post has.", + rollup={"link": "post_snapshots_link", "fn": "countall"}), +] + +#: The POST series. ⚠ D-117 is live on the Instagram twin — `posted_at`/`type` are denormalised onto +#: snapshot rows there and never reconcile with the post row afterwards. That pattern is NOT +#: reproduced: this table carries the measurement and its identity, and asks `ut_tt_posts` for what +#: kind of post it was. A fact that can disagree with its own dimension table is a fact nobody can +#: trust, and the convenience it buys (one fewer hop in a rollup) is not worth a column that can be +#: wrong. +TT_POST_SNAPSHOT_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("post_snapshot_key", "Snapshot"), + tt_field_def("shortcode", "Post id"), + tt_field_def("influencer_key", "Influencer"), + tt_field_def("pulled_at", "Pulled at", "date"), + tt_field_def("views", "Views", "int"), + tt_field_def("likes", "Likes", "int"), + tt_field_def("comments", "Comments", "int"), + tt_field_def("shares", "Shares", "int"), + tt_field_def("saves", "Saves", "int"), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("post_link", "Post", "link", + description="Post record linked to this measurement.", + link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), +] + +#: Comments. Same posture as Instagram's: the SCHEMA and the LINK ship, the CAPTURE is opt-in and +#: defaults OFF (`commentMetrics`), because comments are the single most expensive thing this +#: product can buy and they ingest identifiable third parties who never entered anybody's list +#: (D-24). ⚠ TikTok's `replies` is an ARRAY where ours is an INT count — a name collision, resolved +#: in the mapper by storing `num_replies` and leaving the array in Source data. +TT_COMMENT_FIELDS = [ + tt_field_def("platform", "Platform"), + tt_field_def("comment_key", "Comment"), + tt_field_def("shortcode", "Post id"), + tt_field_def("influencer_key", "Influencer"), + # ⭐⭐ OWNER RULING 2026-08-12, verbatim: *"why isn't any of the comments column actually HAS + # the comments content, fix it."* The text was bought, stored and INVISIBLE — retained whole in + # `source_payload` under the 2026-08-07 superseding ruling (`instagram-capture.md` §4b), but + # promoted to no column, so a person paying per comment record saw only Likes and Replies. + # ⛔ THE COMMENT TEXT ONLY — the commenter's IDENTITY (`commenter_user_name`, `commenter_id`, + # `commenter_url`; the vendor FLAGS the first as PII) stays in `source_payload` and gets no + # column. The ruling names the comments' CONTENT, and promoting a third party's name and + # profile URL into a filterable, exportable column is a different decision that nobody made. + tt_field_def("text", "Comment"), + tt_field_def("commented_at", "Commented at", "date"), + tt_field_def("likes", "Likes", "int"), + tt_field_def("replies", "Replies", "int"), + tt_field_def("source_payload", "Source data", "json"), + tt_field_def("post_link", "Post", "link", + description="Post record linked to this comment.", + link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", + "single": True}), +] + +#: table key -> the field list that defines it. ⭐ DERIVED CONSUMPTION IS THE POINT: `ut_ensure`, +#: the gates and every future runner read THIS rather than naming five constants, so adding a sixth +#: `ut_tt_*` table is one entry instead of a sweep. The Instagram side has no equivalent map, which +#: is exactly why its table names are scattered across the module. +TT_TABLE_FIELDS = { + TT_PROFILE_TABLE: TT_PROFILE_FIELDS, + TT_SNAPSHOTS_TABLE: TT_SNAPSHOT_FIELDS, + TT_POSTS_TABLE: TT_POST_FIELDS, + TT_POST_SNAPSHOTS_TABLE: TT_POST_SNAPSHOT_FIELDS, + TT_COMMENTS_TABLE: TT_COMMENT_FIELDS, +} +#: The human labels a spawned `ut_tt_*` table wears in the database list. +TT_TABLE_LABELS = { + TT_PROFILE_TABLE: "TikTok profiles", + TT_SNAPSHOTS_TABLE: "TikTok profile measurements", + TT_POSTS_TABLE: "TikTok posts", + TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements", + TT_COMMENTS_TABLE: "TikTok comments", +} +#: ⭐⭐ WAVE 31 · OWNER RULING R9 — the `ut_tt_*` children a person may not type records into. +#: Owner, verbatim: *"Tiktok database for post and comments and their snapshots should also be a +#: locked database with the lock icon, exactly like how instagram"*. +#: +#: ⛔ DECLARED ONCE BECAUSE THE DEFECT WAS TWO CALL SITES DISAGREEING WITH A THIRD. `ensure_ig_graph` +#: passes `record_mode=AUTOMATION_RECORD_MODE` for all four Instagram children; TikTok's two spawn +#: sites — `_spawn_presets`' child loop (SAVE) and `_tt_write_tables` (RUN) — each passed +#: `lock_fields=True` and no `record_mode` at all. That is the whole bug: not a missing feature, one +#: argument missing at two places, which is exactly the shape `_tt_write_tables`' own docstring warns +#: about (*"one function, two callers, so the inline and deferred paths cannot answer differently"*). +#: A SET plus a resolver means a sixth `ut_tt_*` table joins by declaration, not by remembering. +#: +#: ⚠ FOUR, NOT THE THREE R9 ENUMERATES, and the fourth is argued rather than assumed. +#: `ut_tt_snapshots` is the profile measurement series — the exact twin of `ut_ig_snapshots`, which +#: IS locked. R9's own comparison is *"exactly like how instagram"*, and the alternative is to leave +#: a machine-append time series that a person can type into, one table away from three that they +#: cannot. That is the fix-applied-to-one-platform-and-not-its-twin shape this very wave is closing +#: elsewhere (D-177(c)). `ut_tt_profile` is deliberately ABSENT: it is the database a human adds +#: handles to, and its Instagram counterpart is not locked either. +#: +#: ⚠ LOCKED DATABASE, NOT READ-ONLY (DESIGN.md §4): `records_mutable` goes False, so records cannot +#: be added/edited/deleted — **adding a FIELD must still work**, and a UI hiding add-field here is a +#: defect, not the intent. +TT_LOCKED_TABLES = frozenset({TT_SNAPSHOTS_TABLE, TT_POSTS_TABLE, TT_POST_SNAPSHOTS_TABLE, + TT_COMMENTS_TABLE}) +#: ⭐ THE INSTAGRAM HALF, NAMED. It was only ever the four keys `ensure_ig_graph`'s loop happens to +#: iterate, which is a set that exists but cannot be ASKED — so nothing could assert that IG and +#: TikTok lock the same shape, and the QA sweep below could not be written at all. +IG_LOCKED_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, IG_POST_SNAPSHOTS_TABLE, + IG_COMMENTS_TABLE}) +#: ⭐⭐ W31 QA — OWNER, VERBATIM (2026-08-13): *"just like Instagram Post database (which is +#: locked), only the IG Profile and TT Profile should be editable."* That is this set plus its +#: complement: every `ut_ig_*`/`ut_tt_*` child is locked, and `ut_ig_profile` / `ut_tt_profile` are +#: the two a person types handles into. Registered into `core.user_tables` at import so the lock is +#: a DECLARATION rather than a flag some past automation run happened to stamp — see that module's +#: `_LOCKED_RECORD_KEYS` for why the stored flag alone left production unlocked. +#: ⛔ THE REGISTRATION ITSELF IS IN `main.py`, NOT HERE, and that is this module's own rule rather +#: than a preference: it imports NOTHING from `core` at module level (see `MAX_UT_ROWS` and +#: `MACHINE_OWNERS`, both local literals "so this module stays dependency-light for the API's boot +#: path"). Adding `import core.user_tables` at line 3570 would put `core` on the import path of +#: every process that touches the engine, to run one line. The composition root wires it, and +#: `verify_automation` asserts that main.py CARRIES that call — a declaration whose registrar is +#: missing is [[flag-shipped-without-its-writer]], which is the defect class this whole fix is in. +LOCKED_CHILD_TABLES = IG_LOCKED_TABLES | TT_LOCKED_TABLES + + +def tt_record_mode(table_key): + """R9's answer for one `ut_tt_*` key — the `record_mode` its `ut_ensure` must carry. + + `""` for anything outside the locked set, which is what `ut_ensure` already treats as "say + nothing about record mode", so an unlocked table is untouched rather than explicitly opened. + """ + return AUTOMATION_RECORD_MODE if str(table_key) in TT_LOCKED_TABLES else "" + + +# --------------------------------------------------------------------------------------------- +# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`. +# --------------------------------------------------------------------------------------------- +# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE. +# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation +# runtime needs that, and the runtime is what this file is for. +# +# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation +# runtime still reaches for after the split — twenty, down from the seventy-one that used to be +# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor +# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and +# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that +# true without those files having to learn where the wire went. +# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked +# that would be green whether or not anything moved. `verify_automation`'s `section_split` +# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING +# system instead of trusting this import line ([[gate-answers-the-wrong-question]]). +# +# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back +# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily, +# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail +# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a +# module-level import, the names it wants already exist and the cycle still resolves. +# ⭐⭐ WAVE 30 · T09 (D-128) — TWO IMPORT BLOCKS NOW, AND THE SPLIT BETWEEN THEM IS THE POINT. +# `connectors_bd` is the SUPPLIER's wire — it serves both platforms and knows neither. `connectors_ig` +# is INSTAGRAM's vocabulary: its dataset ids, its row mappers, its free rungs. Anything that would +# have to change to serve a third platform belongs in the second block, not the first. +# ⚠ `bd_ready` is re-exported through this module ON PURPOSE — `routes_automation` and +# `routes_connectors` both reach `engine.bd_ready()` on the module object, and those files belong to +# other sessions. Dropping it here would 500 two surfaces that never mention a vendor. +from connectors_bd import ( # noqa: E402 + BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, BD_RECORD_PRICE_SPEC, + _bd_deferral, _bd_first_url, _bd_rows, + _first, _ig_int, + bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, + bd_snapshot_progress, depth_refusal, +) +from connectors_ig import ( # noqa: E402 + BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, + _bd_comment, _bd_post_metrics, _bd_profile, + _bd_tagged_location, + DEFERRED_MARK, + apify_profile, + bd_profiles_batch, + ig_handle, public_source, pull_profile, select_post_groups, top_up_views, +) + + +# --------------------------------------------------------------------------------------------- +# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) +# --------------------------------------------------------------------------------------------- +# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's +# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns +# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND +# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower +# Georgia/Florida floral-design company — Fisch Floral's actual market. +# +# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: +# +# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` +# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is +# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the +# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. +# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, +# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, +# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still +# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a +# snapshot that is not ready when it looks. +# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery +# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` +# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is +# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the +# next run collects it. A `partial` that says "still building, the next run collects it" is +# the honest report of exactly what happened. +# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and +# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into +# a wider profile set would silently multiply the enrichment bill AND pollute the +# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich. +# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important +# test died on it. +# +# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE +# the API returns a number, `price: 0` means "not priced" rather than "free", and +# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to +# report. Every estimate below is derived from the PRICING PAGE and says so. + +#: The table a discovery run writes when the tenant has no Instagram profile database yet — the +#: FIRST one, never a second (`discover_default_table` elects an existing one before this is +#: reached). Upsert by handle, so re-finding a profile is free. +#: +#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG +#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said +#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`, +#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept +#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it +#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on +#: any tenant who renamed the table, which is exactly the mismatch the owner hit. +#: +#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three +#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the +#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key +#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant +#: would keep working; there simply is not one. +#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration +#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family. +DISCOVER_TABLE = "ut_ig_profile" +#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels, +#: so changing this can rename nothing that already exists. One constant because it was three +#: copies of the same string literal, and a default spelled three times is one edit from drifting. +DISCOVER_LABEL = "IG profile" +#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at +#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed +#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real +#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. +BD_MAX_RECORDS = 500 +#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). +BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) +BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) + +#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error +#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. +BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", + "array_includes", "not_array_includes", "is_null", "is_not_null") +#: Operators that take NO value (everything else requires one). +BD_NULLARY_OPS = ("is_null", "is_not_null") + +#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** +#: by the API and is therefore absent rather than offered-and-broken). +#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY +#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier +#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the +#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by +#: accident; adding them back is a decision, not a typo. +BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", + "category_name", "business_category_name", "is_business_account", + "is_professional_account", "is_verified", "account", "full_name", + "external_url", "bio_hashtags", "post_hashtags", "profile_url", + "related_accounts", "profile_name", "id", "fbid", "highlights_count") +#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: +#: these are the fields seen carrying values on real CORPUS rows. `category_name` and +#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter +#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". +BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") + +#: ⭐⭐ WAVE 32 · T46 / DEBT D-167 — THE FILTER VOCABULARY HAS A PLATFORM NOW, AND IT COST MONEY +#: NOT TO. `BD_FILTER_FIELDS` above was measured against INSTAGRAM in wave 22; waves 29 and 30 hung +#: a second platform on the same tuple, so a `discover_tiktok` could be built on any of the **16 +#: fields Bright Data's TikTok Profiles dataset does not have** — and `BD_FILTER_LEAD`'s third +#: field, `avg_engagement`, is one of the three the Find panel LEADS with and is absent there. +#: A search on a field the corpus does not carry does not error: it returns nothing, and reads +#: exactly like *"no such creators exist"* after the money has been spent. +#: +#: ⛔ MEASURED, NOT REASONED — W30-B20, against TikTok's 40 declared dataset fields: **5 of the 21 +#: names survive.** They are these. Anything else is refused at the door with the field named, +#: which is `clean_predicates`' existing posture (*"refuses rather than coerces … the alternative +#: turns 'find me verified accounts in Georgia' into 'find me any account' and bills for the +#: difference"*) — now applied per platform rather than per product. +#: ⚠ NOT DERIVED FROM `connectors_tt.normalize_profile`, TEMPTING AS THAT IS. That mapper declares +#: the vendor's READ names on a returned row; these are the names its FILTER validator accepts, and +#: the two vocabularies are not the same thing on either platform (`awg_engagement_rate` reads, +#: `avg_engagement` filters). Deriving one from the other would look rigorous and be wrong. +TT_FILTER_FIELDS = ("followers", "following", "biography", "is_verified", "id") +#: The two of `BD_FILTER_LEAD`'s three that TikTok actually carries. `avg_engagement` is dropped +#: for the reason above — leading with a field the corpus cannot answer is worse than leading with +#: two. +TT_FILTER_LEAD = ("followers", "biography") + + +def filter_fields(kind=""): + """The searchable field names for one discovery kind — `(fields, lead)`. + + ⛔ ONE ACCESSOR, so the ROUTE that publishes the vocabulary and the VALIDATOR that enforces it + cannot come to disagree about it. That divergence is D-167 itself: the route served 21 names + with no platform dimension and `clean_predicates` validated against the same tuple with no kind + parameter, so both halves were consistently wrong together and nothing could notice. + """ + if str(kind or "") == "discover_tiktok": + return TT_FILTER_FIELDS, TT_FILTER_LEAD + return BD_FILTER_FIELDS, BD_FILTER_LEAD + +#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. +#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still +#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not +#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a +#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the +#: scanner still has to walk, so they refine a search and cannot BE one. +BD_NARROWING_FIELDS = frozenset({ + "biography", "account", "full_name", "profile_name", "category_name", + "business_category_name", "external_url", "bio_hashtags", "post_hashtags", + "related_accounts", "id", "fbid", "profile_url"}) +#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the +#: whole corpus, which is breadth wearing a condition's clothes. +BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) +#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed +#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% +#: populated: below it a filter misses more corpus than it matches, and the surface should +#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, +#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never +#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. +#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding +#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. +BD_POPULATED_FIELDS = frozenset({ + "account", "followers", "following", "posts_count", "avg_engagement", "biography", + "external_url", "is_business_account", "is_professional_account", "is_verified", + "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) +BD_MIN_NARROWING = 1 + +# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ────────────────── +# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY +# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what +# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner: +# *"look at each damn schema and only provide toggles operator that make sense"*. +# +# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx` +# defended raw names on the grounds that a refusal names the field exactly as the row does — and +# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI +# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio". +BD_FIELD_LABELS = { + "followers": "Followers", "following": "Following", "posts_count": "Post count", + "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category", + "business_category_name": "Business category", "is_business_account": "Business account", + "is_professional_account": "Professional account", "is_verified": "Verified", + "account": "Handle", "full_name": "Name", "profile_name": "Profile name", + "external_url": "Link in bio", "profile_url": "Profile link", + "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts", + "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID", + "highlights_count": "Story highlight count", +} +#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident; +#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson). +BD_FIELD_HINTS = { + "account": "The @username", + "avg_engagement": "Likes and comments as a share of followers", + "id": "Instagram's internal number for the account. For matching a list you already have", + "fbid": "The linked Facebook id. For matching a list you already have", + "profile_name": "The name shown above the bio, when it differs from the account name", + "related_accounts": "Accounts Instagram suggests alongside this one", +} +#: The KIND decides the comparisons offered and the control drawn for the value. +BD_FIELD_KINDS = { + **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement", + "highlights_count")}, + **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")}, + **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")}, + **{n: "choice" for n in ("category_name", "business_category_name")}, + **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url", + "profile_url", "id", "fbid")}, +} +#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of +#: each list is what `default_operator` must agree with. +BD_OPS_BY_KIND = { + # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second + # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one + # idea — and the second one is the one written in vendor grammar. + "text": ("includes", "not_includes", "=", "is_not_null", "is_null"), + "choice": ("=", "!=", "is_not_null", "is_null"), + "number": (">=", "<=", "=", ">", "<"), + # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean + # is `=` with the other value, written the confusing way. + "boolean": ("=",), + "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"), +} +#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire +#: (it is what the API accepts); nobody has to read it. +BD_OP_LABELS = { + "includes": "contains", "not_includes": "does not contain", + "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of", + ">=": "at least", "<=": "at most", ">": "more than", "<": "less than", + "array_includes": "has any of", "not_array_includes": "does not have", + "is_null": "is empty", "is_not_null": "is not empty", +} +#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into. +BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}] + +#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many +#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start` +#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing +#: the whole search to "match any" (which would also drag Followers into the union — the hole the +#: OR guard closed the same day). +#: +#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not +#: contain floral OR does not contain flower" matches nearly every account alive — a negative over +#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a +#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs +#: them enough to write the AND branch. +BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"}) +MAX_PREDICATE_VALUES = 12 + + +def field_label(name): + """The human name for a searchable field — used by the SURFACE and by every REFUSAL.""" + return BD_FIELD_LABELS.get(name) or str(name or "that field") + + +def field_kind(name): + return BD_FIELD_KINDS.get(name, "text") + + +def ops_for(name): + """The comparisons this field may be asked. A tuple, in offer order.""" + return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"]) + +#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing +#: law, because the two drifted apart and the drift cost a user their first automation. +#: +#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no +#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its +#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS` +#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door +#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in — +#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one +#: sentence of drift, and nothing red anywhere at any point. +#: +#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field +#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks +#: one itself — a client-side default is a second copy of this table, free to disagree with the +#: guard that judges it, which is exactly what happened. +BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"}) +BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"}) +#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a +#: worse question than an equality, and both narrow. +BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"}) + + +def default_operator(name): + """The operator a NEW condition on `name` starts with. + + Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from + saveable rather than two (change the comparison, then type a value) — with the second step + being one the surface never told anyone to take. + """ + kind = field_kind(name) + if kind == "tags": + return "array_includes" + if kind in ("boolean", "choice"): + # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through + # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so + # the default would have been an operator the same module refuses to offer — the exact + # default-versus-guard split this function was written to close, reintroduced one commit + # later by adding a kind. Asserted per field in `verify_automation.py`. + return "=" + if kind == "number": + # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person + # reaching for Followers means, and the guard's sentence is the honest answer when it is + # the ONLY condition present. + return ">=" + return "=" if name in BD_EXACT_FIELDS else "includes" + + +def predicate_narrows(p): + """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" + return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS + and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) + + +def narrowing_refusal(preds, operator="and", kind=""): + """C4's server law: the sentence when a predicate set does not narrow, else ''. One function + so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law) + refuse in the same words. + + ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY + predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a + UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR + followers >= 10000` saved clean and asked the vendor for every account over ten thousand + followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop. + A money wall that a dropdown three rows above it can walk through is not a wall. + + So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows + either way — `all([])` is True, which would have made "no conditions at all" the widest legal + query of the lot. + """ + preds = list(preds or []) + if str(operator or "").lower() == "or": + if preds and all(predicate_narrows(p) for p in preds): + return "" + return ("with Match set to any, every condition has to describe the account itself " + "(Bio, Handle, Name, a hashtag). Matching any means the results are added " + "together, so one follower or yes/no condition widens the whole search. Narrow " + "every condition, or set Match to all") + if any(predicate_narrows(p) for p in preds): + return "" + # ⭐ WAVE 32 · T46 (D-167's neighbour) — THE NETWORK'S OWN NAME. This sentence said + # "Instagram" to somebody building a TIKTOK search, on the refusal they are most likely to + # read, because the string predates the second platform. `discovery_facts` already carries the + # network's name for exactly this class of string, so there is no second literal. + _net = discovery_facts(kind)[0] if kind else PLATFORM_INSTAGRAM + return ("add a condition that describes the account itself. Bio contains, Handle is, a " + f"hashtag. Follower counts and yes/no conditions alone match too much of {_net} " + "to search") + + +def filter_meta(): + """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]` + over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, + R3). + + `defaultOperator` rides here rather than being a client constant for the reason written over + `default_operator`: the client's own default was `is_not_null` for every field, which the + narrowing guard refuses on every field. + """ + return [{"name": n, + "label": field_label(n), + "hint": BD_FIELD_HINTS.get(n, ""), + "kind": field_kind(n), + "populated": n in BD_POPULATED_FIELDS, + "narrowing": n in BD_NARROWING_FIELDS, + "defaultOperator": default_operator(n), + # The comparisons THIS field may be asked, already in plain English and already in + # offer order. A client that filtered a global list by field kind would be a second + # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale. + "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o), + "nullary": o in BD_NULLARY_OPS, + # `in`/`is any of` and the tag operators take a LIST — the control + # has to know that, and deriving it from the token in the client is + # the same second-copy mistake one level down. + "multi": o in BD_MULTI_VALUE_OPS} + for o in ops_for(n)], + "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []} + for n in BD_FILTER_FIELDS] + +#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET. +#: +#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what +#: "unified across tenant" means and what makes the pooled master series joinable at all — two +#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query. +#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET +#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat. +#: +#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in +#: `ut_ig_snapshots`. ONE STORE FOR ONE SERIES: no per-record `json` history column, no second copy +#: of a number that already has a home. +#: ⛔ W29-T34 — THIS LINE USED TO ADD *"which the metric fields already read"*, AND THAT WAS WRONG. +#: A `metric` field reads the PLATFORM-WIDE MASTER, not this tenant's snapshot table: +#: `compute_metric_cells` imports `ig_master` and calls `ig_master.series_for(...)` — a +#: cross-tenant pooled repo in a different HF dataset, which is the whole point of C6's pooling and +#: which no rollup kind can reach. The metric header below says exactly that, so the module was +#: contradicting itself on which store answers the question. +#: ⚠ And the practical finding behind the correction, recorded because it is surprising: a live +#: census of all four tenants (13 tables / 228 fields / 64 store files) found **ZERO** metric fields +#: in existence, while 47 rollups are live. The vocabulary stays by owner ruling; nothing uses it. +#: +#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION, +#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields +#: available to us from Bright Data to be pre-set Fields for us and populated."* +#: +#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule +#: and the next reader will otherwise re-derive it and prune these columns back. It said: a +#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on +#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles, +#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is +#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were +#: captured into the snapshots and were NOT preset columns. +#: +#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be +#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the +#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus; +#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does +#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and +#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a +#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make. +#: Source data retains every field the provider returns. The only separate decision is whether to +#: request the paid full Comments dataset (`commentMetrics`, default OFF). +#: +#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is +#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`). +#: +#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person +#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and +#: written down here rather than inferred from a runner's module name, because the day a second +#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the +#: profile simply appears twice. +#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it. +#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals +#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as +#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names +#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things, +#: which is exactly why both are written down instead of inferred. +PROFILE_SOURCE_IG = "instagram" +#: ⚠ Its TikTok twin, `PROFILE_SOURCE_TT`, is declared UP with the `ut_tt_*` schemas (wave 29): the +#: field lists there are evaluated at import and would not see a constant defined here. + +PLATFORM_INSTAGRAM = "Instagram" +#: ⭐ WAVE 29 — the `ut_tt_*` family stamps this on every row of all five of its tables. (It really +#: was written by nothing until D-9 landed; the note that said so is kept in the log, not here.) +PLATFORM_TIKTOK = "TikTok" +PLATFORM_FACEBOOK = "Facebook" +PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK) + + +def discovery_facts(kind): + """⭐⭐ WAVE 30 · T05 — THE THREE THINGS THAT GENUINELY DIFFER BETWEEN THE DISCOVERY KINDS: + the network's NAME, the database a run writes to when the config names none, and that + database's label. `(platform, table, label)`. + + ⛔ WHY A FUNCTION AND NOT A DICT LITERAL: `DISCOVER_TABLE` and `DISCOVER_LABEL` are declared + several hundred lines BELOW this point, so a dict evaluated here would NameError at import. + A function body resolves at call time and does not care. (`PROFILE_SOURCE_IG`'s note directly + above records the same import-order trap costing a constant its natural home.) + + ⛔ AND WHY IT EXISTS AT ALL. These three facts were written out inline at FIVE call sites — + `clean_config`, `graph`'s discovery arm, `graph`'s `det` dict, `_flow_table` and + `compose_sentence` — and wave 29 taught four of the five about TikTok by not touching them: + they tested the string `"discover_instagram"`, so a stored TikTok automation fell through to + a default meant for Instagram, or to no arm at all. Every one of those misses was SILENT and + every gate stayed green. `DISCOVERY_KINDS` answers *"is this a corpus search?"*; this answers + *"whose?"* — and between them a sixth platform is one tuple entry and one arm here, rather + than a hunt through the file for string comparisons somebody has to think to look for. + + ⚠ DELIBERATELY NOT A PLATFORM REGISTRY. The dataset ids, the field maps and the row mappers + stay in the connector modules that own them. This is the presentation-and-default quartet the + ENGINE needs, and widening it is how it becomes a second `SOURCES` with a longer name. + + ⭐ T06 ADDED THE FOURTH ELEMENT, `spawn_fields` — the field list `_presets_after_write` gives + the target database on SAVE. It belongs here rather than beside that function because it must + equal what the RUN-TIME `ut_ensure` uses, and wave 25's R10 exists precisely to stop those two + disagreeing: columns that change between Save and the first run are two different answers to + "what does this database look like". + ⚠ THE TWO PLATFORMS PASS DIFFERENT-LOOKING LISTS AND THAT IS CORRECT, not an oversight. + Instagram spawns `CANDIDATE_FIELDS` (= the preset set PLUS the discovery bookkeeping) because + its preset list alone is a subset of what its runner writes; TikTok's `TT_PROFILE_FIELDS` + already carries its own bookkeeping columns, so it IS the whole set. Each side is the list its + own runner ensures — which is the rule, rather than "both use the one named CANDIDATE". + """ + if kind == "discover_tiktok": + return (PLATFORM_TIKTOK, TT_PROFILE_TABLE, TT_TABLE_LABELS[TT_PROFILE_TABLE], + TT_PROFILE_FIELDS) + return PLATFORM_INSTAGRAM, DISCOVER_TABLE, DISCOVER_LABEL, CANDIDATE_FIELDS + +#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT. +#: +#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as +#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring +#: `followers` as `int` gives NEW tables a schema every table already in production does not have +#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument +#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason +#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format? +#: change this."* +#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without +#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it +#: would do it silently, because a merged-by-key field list produces no error when two tables +#: disagree about what a column IS. +#: +#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor +#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this +#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction +#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the +#: mapping and in the migration both. See `_pct100`. +PRESET_PROFILE_FIELDS = [ + # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now. + # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different + # people as often as not, so the dedup key is (platform, handle) and never handle alone + # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung + # answered* (`field_def("source", "Read via")`), and one word meaning two things across two + # tables in one product is the drift this whole list exists to prevent. + field_def("platform", "Platform"), + # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING. + # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be + # REPLACED or made as the Handle. in any case the user can add their own record right and edit + # those records."* Two declarations, one field, and they are the same sentence read twice: + # + # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT + # this the primary is an accident of creation order. It is a DECLARATION, not a + # reorder: the stored field order is left alone (saved view orders keep working) + # and the client pins the column to position 0 wherever it sits. That is why this + # fixes tables that already exist without moving a single stored field. + # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to + # (`profile_field_key` step 2), which is what turns "I added a profile to my + # database, how do I enrich it?" from an unanswerable question into a step. It + # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle + # all normalise to the bare handle) and routes a typed value to the SHARED + # definition row rather than one user's overlay (C3-A1) — which is precisely what + # makes the owner's second clause true: a record you add BY HAND is a record the + # engine can read and enrich. + # + # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag + # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on + # anything but `text`. + field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}), + field_def("profile_url", "Profile", "url"), + field_def("full_name", "Name"), field_def("followers", "Followers", "int"), + field_def("following", "Following", "int"), + field_def("avg_engagement", "Avg engagement", "pct"), + field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), + field_def("verified", "Verified", "checkbox"), field_def("category", "Category"), + # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields + # MEASURED ≥60% populated (see the note above). + field_def("posts_count", "Post count", "int"), + field_def("highlights_count", "Story highlight count", "int"), + field_def("is_business", "Business account", "checkbox"), + field_def("is_professional", "Professional account", "checkbox"), + # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would + # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it. + field_def("ig_id", "Instagram id"), + # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction + # (see the block comment above for what that reverses). Types are HONEST from birth, which + # costs nothing here: no table has ever carried these columns, so there is no stored `text` + # definition for the migration to convert — R3's conversion rule applies to the columns that + # already exist, and these are new everywhere. + field_def("business_category", "Business category"), + field_def("is_private", "Private account", "checkbox"), + field_def("bio_hashtags", "Bio hashtags"), + field_def("pronouns", "Pronouns"), + field_def("profile_name", "Profile name"), + field_def("is_joined_recently", "Joined recently", "checkbox"), + field_def("has_channel", "Has channel", "checkbox"), + field_def("partner_id", "Partner id"), + field_def("external_url_title", "Link title"), + # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity. + field_def("fbid", "Facebook id"), + field_def("related_accounts", "Related accounts"), + field_def("country_code", "Country"), + field_def("source_payload", "Source data", "json"), + # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT. + # + # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is + # linked to the profile automatically… and this rollup needs to have formula that we can use + # to calculate things like average Views over last N posts."* Both halves are these five + # columns, and they are PRESETS rather than something a person assembles, because "linked + # automatically" is the instruction — a relation you have to wire up by hand is the feature + # not existing. + # + # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged + # column and then the pinned one, both of which are `handle` on every preset table — so this + # links correctly on a database where the handle column was renamed, and on one where the + # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject + # [[gate-answers-the-wrong-question]] warns about, one layer down. + # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total. + # Two count-like columns with the same label would be unreadable. Caught by the label-collision + # gate rather than on screen, which is what that gate is for. + field_def("posts_link", "Post rows", "link", + description="Post records linked to this profile.", + link={"table": IG_POSTS_TABLE, "on": "influencer_key"}), + field_def("profile_snapshots_link", "Profile history", "link", + description="Profile snapshots linked to this profile.", + link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}), + field_def("post_snapshots_link", "Post measurement rows", "link", + description="Post engagement measurements linked to this profile.", + link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}), + field_def("comments_link", "Comment rows", "link", + description="Comment records linked to this profile.", + link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}), + # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup + # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further + # out in `ut_ig_post_snapshots`. + # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be + # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30 + # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant + # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a + # `limit`, and this is why — "the last 12" has to name what makes one post later than another. + # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours + # because its input was an account-grain constant; with the input now a real per-reel + # measurement the average means what its label says again. + field_def("avg_views_12", "Avg views · last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "views", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "plays", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "likes", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup", + rollup={"link": "posts_link", "field": "comments", "fn": "average", + "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", + "distinctBy": "shortcode"}), + # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO + # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after + # a paid enrich. This counts the post rows actually captured — a different number and a true + # one, which is why it gets its own column and its own label rather than quietly filling + # `posts_count` with something that is not what that field means. + field_def("posts_captured", "Posts captured", "rollup", + rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}), + field_def("profile_reads", "Profile reads", "rollup", + rollup={"link": "profile_snapshots_link", "fn": "countall", + "distinctBy": "snapshot_key"}), + field_def("post_measurements_captured", "Post measurements captured", "rollup", + rollup={"link": "post_snapshots_link", "fn": "countall", + "distinctBy": "post_snapshot_key"}), + field_def("comments_captured", "Comments captured", "rollup", + rollup={"link": "comments_link", "fn": "countall", + "distinctBy": "comment_key"}), + # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING. + # + # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we + # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one + # thing a buyer most wants and the one thing the profile row cannot answer. The competitor + # teardown found the same gap solved by GUESSING from bio words, two buckets deep + # ([[janney-ai-teardown]]). + # + # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place + # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the + # post row, plus the vendor's fuller `location`/`location_details` inside the paid + # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a + # creator's own posts is a far better residency signal than a word in a bio, and it costs a + # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1. + # + # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator + # posting from twelve cities gets a low confidence rather than a confident wrong answer, and + # `location_confidence` is the number a person filters on before trusting the guess. A single + # geotagged post produces NO guess at all — n=1 dressed as a pattern is what + # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain. + field_def("location_guess", "Location (guess)"), + field_def("location_confidence", "Location confidence", "pct"), + # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never + # enriched" and a stale one means "enriched in March", and no cell on the row can tell them + # apart. It is the single field that turns the other fifteen from numbers into measurements. + field_def("enriched_at", "Enriched at", "date"), + # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE. + # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the + # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the + # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying + # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two, + # rewritten each run, so a person reading the profile row can see the recent posts without a + # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2. +] +#: The keys the preset set owns — derived, so a field added above cannot be forgotten here. +PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS) + +#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile +#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what +#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the +#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now +#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to +#: enforce a rule nobody declares any more. +#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" — +#: never as "field number zero". +PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "") +PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "") + +#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the +#: alternative is two lists that describe the same columns and drift one label at a time, which is +#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found +#: them, who for, and a human's decision) rather than about the profile, so they are discovery's +#: and not part of the cross-tenant set. +CANDIDATE_FIELDS = [ + *PRESET_PROFILE_FIELDS, + # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell. + # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner + # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a + # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its + # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what + # changed is that "when did we first see this account" is a DAY, and a day is all anybody + # asks it for. `migrate_ig_field_types()` converts the stored strings. + field_def("found_count", "Times found", "int"), + field_def("first_found", "First found", "date"), + field_def("last_found", "Last found", "date"), + # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it + # describes, which is the first place anybody looks up what this column means. It read: *"the + # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two + # people discovering the same profile each get their own row, their own found_count and their + # own review card."* Wave 26 · R4/R5 retired every clause of that. + # + # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two + # people in one workspace who discover the same profile share ONE row, one `found_count` and + # one history — because they are looking at one company's leads, not two private lists. + # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it + # first. It is no part of the dedup key, and a run must never branch on it. + # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged, + # and the one clause of the old note that was still true. + field_def("created_by", "Found by"), + # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have + # a Field called stage to track the progress of the Automation per Record. We don't need + # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage + # column: every candidate carried BOTH a `stage_` select saying where it was AND a + # boolean saying whether it was kept — two progress fields that could disagree, with no rule + # about which one won. The stage field is the one progress column. Nothing replaces this. +] + + +def _profile_backlink_field(table_key, table_label, profile_key): + """A deterministic reciprocal link from one canonical IG table to one profile database. + + The key includes the target table identity, so ten separate Profile databases can all point + into the same canonical Posts/Comments/history tables without one relation overwriting the + next. The join is derived in both directions and therefore needs no cross-table fan-out write. + """ + digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10] + return field_def( + f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link", + description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.", + link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"}, + ) + + +def _profile_schema_for(bound_key): + """Preset fields for an existing Profile table without inventing a second identity column.""" + out = [] + for field in PRESET_PROFILE_FIELDS: + item = dict(field) + if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY: + item.pop("profile", None) + out.append(item) + return out + + +def _tt_profile_schema_for(bound_key): + """⭐ WAVE 30 · T08 — TikTok preset columns for a database whose profile column is `bound_key`. + + ⛔ NOT `_profile_schema_for` WITH A LIST ARGUMENT, and the difference is not stylistic. That + function walks `PRESET_PROFILE_FIELDS` and only ever removes the flag from `PRESET_FLAG_KEY`, + because on the Instagram side the flag lives on exactly one known column. TikTok's binding may + be ANY column a person named (`profile_field_key` step 1), so the rule here has to be stated + the other way round: **every field that is not the bound one arrives as DATA**, stripped of + both the identity flag and `pinned`. + ⚠ Otherwise a database whose TikTok column is `creator` would gain a rival `handle` carrying + `profile: {source: "tiktok"}` — a SECOND profile column, which `user_tables` refuses at both + write doors, so the whole top-up would be rejected and every preset cell would then be dropped + for want of a column. One shared helper for the save-time and run-time paths, so those two + cannot answer "which columns does a TikTok enrich need" differently. + """ + out = [] + for field in TT_PROFILE_FIELDS: + item = dict(field) + if str(item.get("key") or "") != str(bound_key or ""): + item.pop("profile", None) + item.pop("pinned", None) + out.append(item) + return out + + +def _locked_ig_field(field): + """One canonical Instagram field with the immutable preset declaration attached.""" + item = dict(field) + automation = dict(item.get("automation") or {}) + automation["preset"] = True + item["automation"] = automation + return item + + +def _profile_binding(table): + """The declared Profile identity field, falling back only to the canonical handle.""" + fields = list((table or {}).get("fields") or []) + flagged = next((str(f.get("key") or "") for f in fields + if isinstance(f.get("profile"), dict)), "") + if flagged: + return flagged + return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else "" + + +def _ig_schema_contract(rt, profile_tables=None): + """The complete per-tenant Instagram graph contract, including every Profile backlink. + + The fixed child tables are shared. Profile tables are discovered structurally, so an older + user-named target and the built-in candidate database receive the same preset columns, Links, + Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted. + """ + tables = ut_all(rt) + # ⭐ WAVE 32 · T42 — `_ig_contract_tables`, NOT `_ig_profile_tables`: this function decides + # which databases RECEIVE Instagram's 48 columns, and the owner's rule is that a database gets + # them only if it is used for Instagram. See that function for why the detector stays wider. + profile_keys = sorted(set(profile_tables if profile_tables is not None + else _ig_contract_tables(rt))) + backlinks, wanted, drops = [], {}, { + IG_SNAPSHOTS_TABLE: {"post_hashtags"}, + # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today + # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS + # and fed by the `ig_post_views` capability. Leaving it here would have made the migration + # delete, on every authenticated read, the column the same release just added — the + # drop set and the field list are two halves of ONE contract and must move together. + } + for table_key in profile_keys: + table = tables.get(table_key) or {} + bound = _profile_binding(table) + if not bound: + continue + label = str(table.get("label") or table_key) + backlinks.append(_profile_backlink_field(table_key, label, bound)) + schema = _profile_schema_for(bound) + # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram + # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT + # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION. + # + # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every + # contract key it owns — `rollup` included, by its own note — so a column converted beside + # the contract is re-typed back to `int` on the next reconcile, silently, hours later. + # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the + # schema instead of two that disagree. + # + # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a + # column only where this tenant's own rows show the fold already equals the stored value — + # measured, because a named list would have blanked 221 live cells on nurilab alone + # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts + # fewer columns and loses nothing; as its history fills in, later passes convert more. + derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {})) + wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f + for f in schema] + # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states + # to disagree, so it is retired by key just like the old nested Posts JSON. + drops[table_key] = {"posts", "post_hashtags", "tracked"} + wanted.update({ + IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks], + IG_POSTS_TABLE: [*POST_FIELDS, *backlinks], + IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks], + IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks], + }) + return wanted, drops + + +# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use +# the scalar strings their renderers expect; engagement is the exception because Bright Data's +# 0-1 fraction becomes this product's 0-100 percentage. +_IG_RETYPE_CONVERTERS = { + (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100, +} + + +def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): + """Repair machine-owned IG field declarations in one coalesced store update. + + `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not + sufficient for a canonical relation: a stale `link.table`, rollup function, or type would + survive forever. This pass overwrites the contract keys for the fields we own, preserves + unrelated/user fields, and removes only explicitly retired preset columns plus their cells. + """ + drop_by_table = drop_by_table or {} + wanted_by_table = { + key: [_locked_ig_field(f) for f in fields] + for key, fields in wanted_by_table.items() + } + changed = False + + def _up(cur): + nonlocal changed + cur = cur if isinstance(cur, dict) else {} + for table_key, wanted_fields in wanted_by_table.items(): + table = cur.get(table_key) + if table is None: + continue + wanted = {str(f.get("key")): dict(f) for f in wanted_fields} + drops = set(drop_by_table.get(table_key) or ()) + fields, seen = [], set() + drops = _guarded_drops(table, drops) + for stored in table.get("fields") or []: + key = str(stored.get("key") or "") + if key in drops: + changed = True + continue + desired = wanted.get(key) + if desired is None: + fields.append(stored) + continue + # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A + # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten + # from the shipped contract, `rollup` included, so without this the newly + # unlocked "edit a preset rollup" would save, render, recompute and then revert + # at the next enrichment run — an edit that looks like it worked and undoes + # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp. + if _ut().user_edited(stored): + fields.append(stored) + seen.add(key) + continue + repaired = dict(stored) + for dkey, value in desired.items(): + if dkey != "automation": + repaired[dkey] = value + automation = dict(stored.get("automation") or {}) + automation.update(desired.get("automation") or {}) + repaired["automation"] = automation + # These bags define behaviour, not decoration. If the desired field does not use + # one, a stale bag from an earlier type must not remain attached to it. + for bag in ("link", "rollup", "profile", "pinned"): + if bag not in desired: + repaired.pop(bag, None) + if desired.get("type") not in ("select", "multiselect"): + repaired.pop("options", None) + converter = _IG_RETYPE_CONVERTERS.get((table_key, key)) + if (converter and stored.get("type") in ("text", "", None) + and repaired.get("type") != stored.get("type")): + for row in (table.get("rows") or {}).values(): + if str(row.get(key) or "").strip(): + converted = converter(row.get(key)) + if converted: + row[key] = converted + changed = True + if repaired != stored: + changed = True + fields.append(repaired) + seen.add(key) + for key, desired in wanted.items(): + if key not in seen and not any(str(f.get("key") or "") == key for f in fields): + fields.append(desired) + changed = True + if drops: + for row in (table.get("rows") or {}).values(): + for key in drops: + if key in row: + row.pop(key, None) + changed = True + table["fields"] = fields + return cur + + # Avoid a commit when the declarations are already exact. + snapshot = ut_all(rt) + needs = False + for table_key, wanted_fields in wanted_by_table.items(): + if table_key not in snapshot: + continue + table = snapshot.get(table_key) or {} + by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []} + # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip + # the commit when nothing would change; a spared `tracked` column left in the drop set + # answers "yes, work to do" on every single call, forever, and the schema pass would + # rewrite the table on every authenticated read without ever changing a byte. + drops = _guarded_drops(table, drop_by_table.get(table_key) or ()) + if drops & set(by_key): + needs = True + break + for desired in wanted_fields: + stored = by_key.get(str(desired.get("key") or "")) + mismatch = stored is None + if stored is not None: + for key, value in desired.items(): + if key == "automation": + if any((stored.get("automation") or {}).get(k) != v + for k, v in value.items()): + mismatch = True + break + elif stored.get(key) != value: + mismatch = True + break + if not mismatch: + stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired) + mismatch = any(k in stored for k in stale_bags) + if (not mismatch and desired.get("type") not in ("select", "multiselect") + and "options" in stored): + mismatch = True + if mismatch: + needs = True + break + if needs: + break + if needs: + rt.update(UT_STORE_KEY, _up, flush="sync") + return changed + + +def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""): + """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys. + + Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new + Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect + each Profile database to the same Posts, Comments, profile-history, and post-history stores. + """ + profile = ut_get(rt, profile_table) if profile_table else None + bound = str(profile_field or "").strip() + if profile is not None and not bound: + bound = next((str(f.get("key") or "") for f in profile.get("fields") or [] + if isinstance(f.get("profile"), dict)), "") + profile_label = str((profile or {}).get("label") or profile_table) + backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \ + if profile_table and profile is not None and bound else [] + + # ⭐ WAVE 32 · T41 — DERIVED from `IG_TABLE_FIELDS`/`IG_TABLE_LABELS`, which used to be this + # literal. The backlinks stay HERE because they are per-tenant, per-profile-database facts; the + # schema is a module constant. Splitting it that way is what let A's delivery sweep ask this + # module for Instagram's child set instead of hand-typing a fifth copy of these four names. + graph = {key: (IG_TABLE_LABELS[key], [*fields, *backlink]) + for key, fields in IG_TABLE_FIELDS.items()} + keys = {} + for key, (label, fields) in graph.items(): + keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag, + record_mode=AUTOMATION_RECORD_MODE, lock_fields=True) + + if profile_table and profile is not None and bound: + profile_fields = _profile_schema_for(bound) + ut_ensure(rt, profile_label or profile_table, profile_fields, username, + key=profile_table, flow_tag=flow_tag, lock_fields=True) + # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older + # Profile database and IG candidates on the same universal schema while all of them point to + # the same four children. Existing flow provenance is merged, never replaced. + wanted, drops = _ig_schema_contract(rt) + _reconcile_ig_graph_fields(rt, wanted, drops) + return keys + + +# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ─────────────── +# +# Two changes land on data that already exists, and neither is optional once the field defs move: +# +# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a +# `date` column and a "20872" string in an `int` column are the split-schema the old +# "types are text on purpose" note correctly feared. Re-typing without converting is worse +# than not re-typing. +# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as +# `(handle, created_by)` DUPLICATES must be merged rather than left to collide. +# +# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a +# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would +# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a +# column is converted only while its STORED definition still says `text`; the moment it says `pct` +# the work is done and re-running is a no-op. That makes the migration safe to call on every boot, +# which is what it is for. +# +# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice. +# `observed_categories` named its two source tables and went silently empty the day wave 25 let a +# user point an automation somewhere else. Naming tables here would leave exactly those user-named +# databases on the old schema, which is the same bug wearing a migration's clothes. + +#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset +#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told. +MIGRATE_MIN_PRESET_FIELDS = 3 +#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/''). +_MIGRATE_CONVERT = { + "first_found": _day, "last_found": _day, "enriched_at": _day, + "avg_engagement": _pct100, +} + + +#: ⭐⭐ WAVE 30 · T08 — THE COLUMNS THAT CAN ONLY BE TIKTOK'S, DERIVED rather than typed out, so +#: that adding a column to either declaration keeps this set correct instead of quietly emptying +#: it. Eight today: `tt_id`, `like_engagement`, `comment_engagement`, `likes_received`, `region`, +#: `predicted_lang`, `account_created_at`, `source`. +TT_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in TT_PROFILE_FIELDS) + - frozenset(PRESET_PROFILE_KEYS) + - frozenset(f["key"] for f in CANDIDATE_FIELDS)) + + +def _is_tt_profile_table(tbl): + """⭐⭐ WAVE 30 · T08 — is this database a TIKTOK profile table? + + ⛔ THE DEFECT THIS EXISTS TO CLOSE WAS LIVE AND SHIPPED, and it converted a customer's TikTok + database into an Instagram one on the SECOND write to it. MEASURED: a `ut_tt_profile` spawned + by W30-T06 comes out correct — `handle` carrying `profile: {source: "tiktok"}` — and one more + `ut_ensure` on that table (a re-save, the discovery runner's own ensure, the next run) leaves + it carrying `profile: {source: "instagram"}` and the description *"Instagram username without + the @ symbol"*. `profile_field_key(..., source="tiktok")` then answers `""` forever, so every + TikTok enrich step on that database reports UNBOUND, and every row gets stamped + `platform: Instagram` — which is half of W26/R4's `(platform, handle)` identity, so two + different people's accounts merge under one key with nothing red anywhere. + ⚠ It hid because the migration SKIPS a table that does not exist yet, so the spawn itself is + always clean and only the second write is not. A gate that creates and asserts once cannot see + it; the assertion has to survive a second `ut_ensure`. + + ⭐ TWO LEGS, and the second is the one that still works after the first has been eaten: + 1. the table SAYS SO — a column declaring `profile: {source: "tiktok"}` is the database's own + statement of which network it is about, and it is the same declaration `profile_field_key` + reads, so there is one answer to "whose profile table is this"; + 2. it declares a column only TikTok has. Needed because leg 1 is exactly what the defect + destroys: on a table already corrupted in production, the flag now says Instagram, and a + detector resting on it alone would agree with the corruption and keep re-applying it. + """ + fields = (tbl or {}).get("fields") or [] + for f in fields: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_TT: + return True + return bool({f.get("key") for f in fields} & TT_ONLY_PROFILE_KEYS) + + +#: ⭐⭐ WAVE 32 · T40 — THE MIRROR OF `TT_ONLY_PROFILE_KEYS`, and it is the subject of owner item 1: +#: the profile columns that can only ever be INSTAGRAM's. 26 today. Derived by the same subtraction +#: in the other direction, so a column moved between the two declarations changes both sets at once +#: rather than leaving one of them quietly asserting a stale fence. +#: ⚠ IT IS DERIVED FROM `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS` — discovery's bookkeeping +#: columns (`found_count`, `first_found`, `last_found`, `created_by`) are declared on BOTH platforms, +#: so subtracting the preset list alone would report four keys as Instagram-only that TikTok's own +#: table has carried since wave 29. +IG_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in CANDIDATE_FIELDS) + - frozenset(f["key"] for f in TT_PROFILE_FIELDS)) + +#: ⭐⭐ WAVE 32 · T42 — the preset columns NOBODY TYPES: every `link` and `rollup` in the Instagram +#: preset set. Derived from the declaration, so adding a rollup to the contract widens this for +#: free. `_carries_ig_presets` uses it as the leg that survives when the `automation.preset` stamp +#: is absent — a hand-made creator list has `handle` and `followers`; it does not have `posts_link`. +PRESET_MACHINE_ONLY_KEYS = frozenset( + f["key"] for f in PRESET_PROFILE_FIELDS if f.get("type") in ("link", "rollup")) + + +def platform_schema(platform): + """⭐⭐ WAVE 32 · T41 — ONE ASKABLE DECLARATION OF WHAT A PLATFORM'S DATABASES ARE. + + Returns, for `PLATFORM_INSTAGRAM` or `PLATFORM_TIKTOK`: + + {"platform", "profile_table", "profile_label", "profile_fields", "preset_keys", + "only_keys", "locked_tables", "children": {key: {"label", "fields", "record_mode"}}} + + ⛔ **WHY IT IS A FUNCTION AND NOT A DICT LITERAL — the same import-order trap `discovery_facts` + records twenty lines from its own declaration.** `CANDIDATE_FIELDS` and `DISCOVER_TABLE` are + declared HUNDREDS of lines below `TT_TABLE_FIELDS`, so any module-level dict spanning both + platforms NameErrors at import. A function body resolves at call time and does not care. That is + not a style preference here; it is the reason two earlier attempts at a platform registry in this + file ended up as five inline copies instead. + + ⛔⛔ **AND IT IMPORTS NOTHING FROM `core`, WHICH IS THE ACTUAL CONTRACT WITH `main.py`.** This + module is deliberately dependency-light on the API's boot path (`MAX_UT_ROWS`, `MACHINE_OWNERS` + and `LOCKED_CHILD_TABLES` all say so in their own notes), and the REGISTRAR — the thing that + turns these names into `core.user_tables` state — lives in the composition root. So this returns + plain lists and dicts: a caller can `register_locked_records(...)`, `ut_ensure(...)` or diff it + against a live tenant without `core` ever appearing on the engine's import path. `verify_ + automation` asserts that, by importing this module in a CLEAN interpreter and checking + `core.user_tables` is absent from `sys.modules` afterwards — [[artifact-with-no-importer]] in + reverse: a declaration whose registrar cannot reach it is the same defect as a registrar with + nothing to register. + + ⚠ FIELD DICTS ARE COPIED ONE LEVEL. Every existing call site passes the module lists straight + into `ut_ensure`, which is safe only because nobody has yet appended to one; a boot sweep looping + over both platforms is exactly the caller that would. The copy costs a few hundred dict + constructions once per boot and removes the whole question. + + ⚠ `profile_fields` is the platform's PRESET set as spawned — Instagram's is `CANDIDATE_FIELDS` + (the preset set PLUS discovery's bookkeeping), not `PRESET_PROFILE_FIELDS`, because that is what + `discovery_facts` actually hands the spawn. Two answers to "Instagram's field set" is the drift + this function exists to end, so it gives the one the product uses. + """ + name = str(platform or "") + if name == PLATFORM_TIKTOK: + children = {k: {"label": TT_TABLE_LABELS[k], "fields": [dict(f) for f in TT_TABLE_FIELDS[k]], + "record_mode": tt_record_mode(k)} + for k in sorted(TT_TABLE_FIELDS) if k != TT_PROFILE_TABLE} + return { + "platform": PLATFORM_TIKTOK, + "profile_table": TT_PROFILE_TABLE, + "profile_label": TT_TABLE_LABELS[TT_PROFILE_TABLE], + "profile_fields": [dict(f) for f in TT_PROFILE_FIELDS], + "preset_keys": tuple(f["key"] for f in TT_PROFILE_FIELDS), + "only_keys": TT_ONLY_PROFILE_KEYS, + "locked_tables": TT_LOCKED_TABLES, + "children": children, + } + if name == PLATFORM_INSTAGRAM: + children = {k: {"label": IG_TABLE_LABELS[k], "fields": [dict(f) for f in fields], + # ⚠ NOT `tt_record_mode`'s twin by accident: `ensure_ig_graph` passes + # `AUTOMATION_RECORD_MODE` for all four unconditionally, and + # `IG_LOCKED_TABLES` is exactly those four. Derived from the SET so that + # unlocking one there unlocks it here, rather than from the constant. + "record_mode": (AUTOMATION_RECORD_MODE if k in IG_LOCKED_TABLES else "")} + for k, fields in sorted(IG_TABLE_FIELDS.items())} + return { + "platform": PLATFORM_INSTAGRAM, + "profile_table": DISCOVER_TABLE, + "profile_label": DISCOVER_LABEL, + "profile_fields": [dict(f) for f in CANDIDATE_FIELDS], + "preset_keys": tuple(f["key"] for f in CANDIDATE_FIELDS), + "only_keys": IG_ONLY_PROFILE_KEYS, + "locked_tables": IG_LOCKED_TABLES, + "children": children, + } + raise ValueError(f"no schema is declared for platform {platform!r}") + + +def definition_platforms(defn): + """Which networks ONE automation uses — `{"Instagram"}`, `{"TikTok"}`, both, or empty. + + ⭐ WAVE 32 · T42. Three ways a definition names a network, and all three count, because the + owner's rule is about what a database is USED FOR rather than about how the automation was + built: its discovery KIND (a corpus search is a network by construction), its enrich ACTIONS + (a `plain` flow that enriches is using that network), and the retired `field_instagram` kind, + which is uncreatable but alive on stored definitions (D-65) and still runs Instagram. + """ + defn = defn or {} + kind = str(defn.get("kind") or "") + out = set() + if kind in DISCOVERY_KINDS: + out.add(discovery_facts(kind)[0]) + if kind == "field_instagram": + out.add(PLATFORM_INSTAGRAM) + actions = (defn.get("flow") or {}).get("actions") or [] + if _actions_of_kind(actions, "enrich_instagram"): + out.add(PLATFORM_INSTAGRAM) + if _actions_of_kind(actions, "enrich_tiktok"): + out.add(PLATFORM_TIKTOK) + return out + + +def automation_platforms_by_table(rt, definitions=None): + """⭐⭐ WAVE 32 · T42 (owner item 1, third clause) — WHICH NETWORKS EACH DATABASE IS USED FOR. + + `{table_key: {"Instagram", "TikTok"}}`, over every stored automation. The owner's words are the + whole specification: *"if a user decide to use one database for both Tiktok and Instagram + scraping, only then can the pre-set Fields can exist in the same database."* — so the question a + schema pass has to be able to ask is *"which networks target THIS database"*, and until now + nothing in this module could answer it. + + ⛔ ASKABLE AND `core`-FREE, for `platform_schema`'s reason one function up: it reads only the + automations bucket through `all_definitions`, returns plain sets, and the T41 subprocess probe + covers it — a second askable declaration that quietly needed `core` would make the import-purity + guarantee mean "true for the things we remembered". + + ⚠ `definitions` is a LENT list, same contract as `ut_ensure`'s `tables=`: a caller already + holding the bucket must not pay for a second deep copy of it. + """ + # ⚠ `all_definitions` answers a DICT keyed by id, not a list — and iterating it directly hands + # every consumer a STRING that looks like a definition until the first `.get`. Normalised here + # so a caller lending its own list gets the same treatment. + defs = all_definitions(rt) if definitions is None else definitions + defs = list(defs.values()) if isinstance(defs, dict) else list(defs or []) + out = {} + for defn in defs: + if not isinstance(defn, dict): + continue + table = str(_flow_table(defn) or (defn.get("config") or {}).get("targetTable") or "") + if not table: + continue + found = definition_platforms(defn) + if found: + out.setdefault(table, set()).update(found) + return out + + +def platform_schemas(): + """Both declarations, in `PLATFORMS` order — what a boot sweep loops over. + + ⚠ `PLATFORM_FACEBOOK` is in `PLATFORMS` (it is a value the `platform` COLUMN may hold) and has + no schema, so this filters rather than raising: a caller sweeping every declared platform must + not be broken by a vocabulary entry that names no databases. `platform_schema` still raises for + it, which is the right answer to somebody ASKING for a schema that does not exist. + """ + out = [] + for name in PLATFORMS: + try: + out.append(platform_schema(name)) + except ValueError: + continue + return out + + +def _ig_profile_tables(rt, tables=None): + """Every `ut_*` table in this tenant that looks like an Instagram profile table. + + ``tables`` is the optional lent snapshot (W29-T01): read-only walk, so a caller that already + holds the bucket must not pay for a second deep copy of it. + + ⛔ WAVE 30 · T08 — AND THE TEST IS NOW EXCLUSIVE, because "looks like" was network-blind and a + TikTok profile table looks EXACTLY like one: 18 of `ut_tt_profile`'s 30 columns are Instagram + preset keys, and the bar here is `handle` plus three. Every caller of this function then treats + the match as an Instagram table — `migrate_ig_tables` rewrites its declarations to Instagram's + contract, and `discover_default_table` offers it as the default target for an Instagram search. + """ + out = [] + for key, tbl in sorted(((ut_all(rt) if tables is None else tables) or {}).items()): + keys = {f.get("key") for f in (tbl or {}).get("fields") or []} + if "handle" not in keys or len(keys & set(PRESET_PROFILE_KEYS)) < MIGRATE_MIN_PRESET_FIELDS: + continue + if _is_tt_profile_table(tbl): + continue + out.append(key) + return out + + +def _ig_contract_tables(rt, tables=None): + """⭐⭐ WAVE 32 · T42 — the tables Instagram's preset CONTRACT may be applied to. + + ⛔⛔ THIS IS A DIFFERENT QUESTION FROM `_ig_profile_tables` AND CONFLATING THEM COST FIVE RED + CHECKS. That function answers *"does this database LOOK like an Instagram profile table?"* and + two callers need exactly that: `discover_default_table`, which elects an existing table so a new + automation does not mint a second empty one — and it necessarily runs BEFORE any automation + targets that table, so a targeted-only test makes it elect nothing forever — and the detector's + own negative control. THIS function answers *"may we write Instagram's 48 columns into it?"*, + which is the owner's rule and a strictly narrower set. One normalizer answering two questions is + how a fix in one becomes a silent regression in the other ([[one-question-two-normalizers]]). + + A table qualifies when it looks like one AND any of: + - it is `ut_ig_profile`, Instagram's canonical database by declaration; + - it ALREADY carries the contract (`_carries_ig_presets`) — narrowing must never orphan a + table that has the columns, or its links and rollups silently stop being repaired; + - an INSTAGRAM automation targets it, which is the owner's rule stated in code. + + ⚠ The automations bucket is read LAZILY, on the first candidate that needs the question asked — + a tenant whose tables all already carry the contract pays nothing, on a path this wave is + otherwise trying to make faster. + """ + targets = None + out = [] + for key in _ig_profile_tables(rt, tables=tables): + tbl = (ut_all(rt) if tables is None else tables).get(key) or {} + if key != DISCOVER_TABLE and not _carries_ig_presets(tbl): + if targets is None: + targets = automation_platforms_by_table(rt) + if PLATFORM_INSTAGRAM not in targets.get(key, set()): + continue + out.append(key) + return out + + +def _preset_owned(field): + """Is this column MACHINE-authored, i.e. may `retract_foreign_presets` delete it? + + ⛔ MODULE-LEVEL, NOT A CLOSURE, AND THAT IS WHY IT MOVED. It was a nested `_machine` and the + negative control written against it came out BLIND: nothing outside the function could break + the one guard standing between a boot-time repair and a customer's own column, so the control + ended up patching the key SETS instead and proved something else. A guard a control cannot + reach is a guard nobody has tested [[gate-negative-control]]. + + Two legs, the second for tables that predate the stamp: the `automation.preset` mark that + `ut_ensure(lock_fields=True)` and `_locked_ig_field` both write, or a key that is a preset + LINK/ROLLUP — nobody types `posts_link` by hand. + """ + automation = (field or {}).get("automation") + if isinstance(automation, dict) and automation.get("preset") is True: + return True + return str((field or {}).get("key") or "") in PRESET_MACHINE_ONLY_KEYS + + +def _filled_count(rows, key): + """How many rows carry a non-blank value in `key` — the number that decides REPORT vs DELETE. + + ⛔ ALSO MODULE-LEVEL FOR ITS CONTROL'S SAKE. This one number is the whole of W30/R6's second + sentence inside `retract_foreign_presets`: a foreign column with cells in it is reported, never + deleted. A control that cannot make this lie cannot prove the report is doing anything. + """ + return sum(1 for r in (rows or {}).values() + if isinstance(r, dict) and str(r.get(key) or "").strip()) + + +def retract_foreign_presets(rt, log=print): + """⭐⭐ WAVE 32 · T46 / DEBT D-152 — REMOVE THE PRESET COLUMNS OF A NETWORK A DATABASE IS NOT + USED FOR. The RETRACTION half of `_ig_contract_tables`' recruitment rule, and A's boot sweep + (`W32-T07`) calls it once per tenant. + + ⛔ WHY IT HAS TO EXIST AT ALL, and this is the wave's own thesis one layer down. W30-T08 fixed + the DETECTOR, so the corruption cannot recur — and nothing undoes it. MEASURED on a fixture: a + `ut_tt_profile` corrupted before that fix keeps ALL 26 Instagram-only columns and its + `profile: {source: "instagram"}` flag through `ut_ensure`, `migrate_ig_tables` AND the discovery + spawn. Three write paths, zero repairs. *"The corruption cannot recur"* and *"the corruption is + gone"* are different claims [[a-migration-that-runs-on-the-next-write]]. + + Returns `{"tables", "columns", "cells", "flags", "kept"}`. **`kept` is the important one** — + see guarantee 4. + + THE FOUR GUARANTEES, in the order they matter: + 1. **Only machine-authored columns go.** A field must be stamped `automation.preset: True` + (or be a link/rollup this contract owns) to be eligible. A column a PERSON typed is never + touched, whatever it is named — the D-152 row calls this *"a VALUE BACKFILL over live + customer rows"*, and that is the line it must not cross. + 2. **Only the EXCLUSIVE sets.** `IG_ONLY_PROFILE_KEYS` (26) and `TT_ONLY_PROFILE_KEYS` (8), + both derived. A key both platforms declare — `platform`, `handle`, `followers`, the four + discovery bookkeeping columns — is shared vocabulary and is never a foreign column. + 3. **Idempotent.** A second call finds nothing and writes nothing, so it is safe on every + boot, which is what makes it callable from `main.py` rather than being a script somebody + has to remember to run (the whole reason D-201 is still open). + 4. ⛔ **A FOREIGN COLUMN THAT HOLDS DATA IS REPORTED, NOT DELETED.** W30/R6's second sentence: + a limit that cannot be removed is reported with its cause. Deleting a customer's populated + cells to make a grid look clean is not a repair, and this is precisely the *"deserves its + own supervision"* clause of D-152. `kept` names every such column with its row count, so + the caller can print it and a person can decide. + + ⚠ THE FLAG IS REPAIRED TOO, and it is the half D-152 names explicitly: a TikTok profile table + whose `handle` was re-declared `profile: {source: "instagram"}` answers `""` to + `profile_field_key(..., source="tiktok")` forever, so every TikTok enrich on it reports UNBOUND. + Repaired only where `_is_tt_profile_table` recognises the table by its OWN TikTok-only columns, + i.e. by the leg the corruption cannot have eaten. + """ + stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} + targets = automation_platforms_by_table(rt) + exclusive = {PLATFORM_INSTAGRAM: IG_ONLY_PROFILE_KEYS, + PLATFORM_TIKTOK: TT_ONLY_PROFILE_KEYS} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for key, tbl in sorted(cur.items()): + if not isinstance(tbl, dict): + continue + fields = tbl.get("fields") or [] + keys = {str(f.get("key") or "") for f in fields} + # Only PROFILE databases are in scope: `handle` plus a real slice of either preset + # vocabulary. A posts or comments table shares no profile columns and is never a + # candidate; a database with nothing in common with either is not one either. + if "handle" not in keys: + continue + tt = _is_tt_profile_table(tbl) + used = set(targets.get(key) or ()) + if not used: + # Nothing targets it. Its own declaration is then the only statement of what it is + # for — and a table that says TikTok is not an Instagram table, whatever columns a + # past migration wrote into it. + used = {PLATFORM_TIKTOK} if tt else {PLATFORM_INSTAGRAM} + foreign = set() + for platform, own in exclusive.items(): + if platform not in used: + foreign |= set(own) + if not foreign: + continue + rows = tbl.get("rows") or {} + drop, touched = [], False + for f in fields: + fkey = str(f.get("key") or "") + if fkey not in foreign or not _preset_owned(f): + continue + filled = _filled_count(rows, fkey) + if filled: + # Guarantee 4 — REPORTED, never deleted. + stats["kept"].append({"table": key, "column": fkey, "rows": filled}) + continue + drop.append(fkey) + if drop: + tbl["fields"] = [f for f in fields if str(f.get("key") or "") not in drop] + for r in rows.values(): + if not isinstance(r, dict): + continue + for fkey in drop: + if fkey in r: + r.pop(fkey, None) + stats["cells"] += 1 + stats["columns"] += len(drop) + touched = True + # ⚠ THE FLAG, and only on a table TikTok's own columns still identify. + if tt: + for f in tbl.get("fields") or []: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG: + f["profile"] = {**p, "source": PROFILE_SOURCE_TT} + stats["flags"] += 1 + touched = True + if touched: + stats["tables"] += 1 + return cur + + # ⚠ The precheck is the WHOLE mutation, run against a snapshot, so the common case (nothing to + # do) spends no commit — the same posture `_reconcile_ig_graph_fields` takes, and the reason + # this is safe to call on every boot. + probe = copy.deepcopy(ut_all(rt)) + _up(probe) + if not (stats["columns"] or stats["flags"]): + if stats["kept"]: + log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they " + f"hold data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" + for k in stats["kept"])) + return stats + stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} + rt.update(UT_STORE_KEY, _up, flush="sync") + log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and " + f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)") + if stats["kept"]: + log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they hold " + f"data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" + for k in stats["kept"])) + return stats + + +def _carries_ig_presets(tbl): + """Has this table ALREADY been given Instagram's preset contract? + + ⭐⭐ WAVE 32 · T42 — THE RECRUIT/REPAIR LINE, AND IT RESTS ON A STAMP THE CODE ALREADY WRITES. + `ut_ensure(lock_fields=True)` and `_locked_ig_field` both set `automation: {preset: True}` on + every preset column, and `migrate_ig_tables` BACKFILLS it on a table that predates the stamp + (MEASURED: an unstamped 44-column fixture comes out 44/44 stamped after one pass). So "already + carries the contract" is a declaration to read, not a threshold to invent — and inventing one + was the alternative, because `MIGRATE_MIN_PRESET_FIELDS` is 3 and cannot tell a recruited table + from a hand-made creator list that happens to have `handle`, `followers` and `bio`. + + ⛔ WHY THE LINE EXISTS AT ALL (owner item 1's third clause, MEASURED): a hand-made 5-column + database that NO automation targets went to **45 columns** the moment an Instagram automation + aimed at a DIFFERENT database was SAVED — because `_ig_schema_contract` walks every + structurally-matching table in the tenant and hands each one the full 48-column contract. The + owner's rule is that the preset fields may live in a database only if it is used for that + scraping; recruitment by resemblance is the opposite of that rule. + + ⚠ NARROWING NEVER DROPS A COLUMN. `_reconcile_ig_graph_fields` deletes only the fixed retired-key + `drops` set, so a de-recruited table keeps every field and cell it has — it stops being ADDED to + and REPAIRED, which is the whole of the change. Asserted on a fixture rather than reasoned. + ⚠ AND THE COST IS PAID ONLY WHEN IT HAS TO BE: `_ig_profile_tables` reads the automations bucket + lazily, on the first candidate that is neither stamped nor the canonical `ut_ig_profile` — so a + tenant whose tables are all genuinely Instagram's pays nothing on a path this wave is otherwise + trying to make faster. + + ⛔⛔ TWO LEGS, AND THE SECOND IS THE ONE THAT KEEPS THIS FROM BREAKING A LIVE TENANT. The stamp + is written by the very passes this predicate now gates, so a table recruited BEFORE the stamp + existed and targeted by no surviving automation would be de-recruited and never stamped — + silently losing its rollup/link repair, which is the slowest and worst failure available here + (MEASURED on a stripped fixture: 44 columns, 0 stamps, 0 repaired). So a table ALSO counts as + already-carrying when it declares one of the preset set's LINK or ROLLUP columns. Those are + machine-authored by construction — nobody types `posts_link` or `avg_views_12` into a hand-made + creator list — so the second leg is derived from the contract itself, not a threshold somebody + picked. [[gate-and-nc-must-not-share-a-binding]]'s cousin: a discriminator written by the thing + it discriminates needs an independent second leg, exactly as `_is_tt_profile_table` needed one. + """ + fields = (tbl or {}).get("fields") or [] + for f in fields: + if str(f.get("key") or "") not in PRESET_PROFILE_KEYS: + continue + automation = f.get("automation") + if isinstance(automation, dict) and automation.get("preset") is True: + return True + return bool({str(f.get("key") or "") for f in fields} & PRESET_MACHINE_ONLY_KEYS) + + +def discover_default_table(rt, tables=None): + """Which database should a discovery automation write to when nobody has said? (2026-08-10) + + Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there. + We only need ONE IG profile so it's not confusing."* + + ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_ + leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the + label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own + note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is + a constant rather than a question, a tenant that already had an Instagram profile database got + a SECOND, empty one the first time a discovery automation was saved without a target. + MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside + `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them + pure confusion. + + So the default becomes a question asked of the tenant: + + exactly one profile database (excluding the fallback) -> that one + several, exactly one of which holds rows -> the one in use + several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE) + + ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and + picking one silently would write a paid discovery run into a database the user did not name — + worse than the empty table this exists to prevent. "" means "no opinion", and the caller's + existing fallback stands, which is exactly today's behaviour for that case. + ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset + schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the + empty table would keep re-electing it and nothing would ever change. + + ⭐ WAVE 29 (W29-T01) — ONE READ, NOT TWO. This function read the whole `user_tables` bucket + here and `_ig_profile_tables` read it AGAIN one line below: two 35.8 MB-ceiling deep copies to + answer one question, on a route the automation surface calls on every open. The election is + now computed from a single snapshot, which is also the only way it can be internally + consistent — the two reads could disagree with each other under a concurrent write. + ``tables`` lets `GET /automations` lend the copy it already holds; the answer is still derived + fresh on every call, so there is no memo to invalidate when a database is created or deleted. + """ + try: + tables = (ut_all(rt) if tables is None else tables) or {} + found = [k for k in _ig_profile_tables(rt, tables=tables) if k != DISCOVER_TABLE] + except Exception: # noqa: BLE001 + return "" + if len(found) == 1: + return found[0] + if len(found) > 1: + used = [k for k in found if (tables.get(k) or {}).get("rows")] + if len(used) == 1: + return used[0] + return "" + + +def _apply_discover_default(rt, raw): + """Fill in a discovery automation's target BEFORE the pure validator invents one. + + ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that + turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure + validator with a two-argument contract every gate and route depends on). Resolving only at the + RUN sites would be cosmetic: the literal is written into the STORED config at save time, so + `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted. + `create`/`patch` are the one pair that both know the tenant and sit above that validator. + ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save + stored — is the user's choice and is never rewritten. + """ + cfg = raw.get("config") + if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip(): + return raw + default = discover_default_table(rt) + if not default: + return raw + return {**raw, "config": {**cfg, "targetTable": default}} + + +def _vestigial_name_field(fields, rows): + """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half. + + `user_tables.create()` mints EXACTLY ONE column on a hand-made database — + `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the + automation's columns after it, never reordering. So on every Instagram database somebody + created before pointing an automation at it, column 1 is a `Name` nothing will ever write. + + ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative + because this is the one branch in the migration that DESTROYS something. A column carrying any + declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with + a value in any row is somebody's data. Either way it survives as an ordinary column and merely + stops being the locked primary, which the `pinned` half achieves on its own. + + ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted + `custom__` (`buildOverlayField`) and every machine one comes from a `field_def` + list, so nothing but `create()`'s default can hold this key. The label and type are checked + anyway — a renamed or retyped column is a column someone touched on purpose. + """ + f = next((f for f in fields if f.get("key") == "name"), None) + if f is None or f.get("label") != "Name" or f.get("type") != "text": + return None + if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")): + return None + if any(str(r.get("name") or "").strip() for r in rows.values()): + return None + return f + + +def _merge_candidates(rows): + """C3's merge rule, applied to rows now colliding on `(platform, handle)`. + + Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell + takes the first non-blank, preferring the most recently enriched row. + + ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and + board cards; minting a new one would orphan all of them, so a merge picks a survivor rather + than creating one. + """ + groups, keepers = {}, {} + for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))): + r = rows[rid] + h = str(r.get("handle") or "").strip() + if not h: + # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a + # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a + # manual handle in my 'Beauty influencer leads' database, after an automation run, it + # gets deleted?"* + # + # A row you add by hand is born blank and stays blank until you type into it, so the + # window is not narrow — it is every row, from creation until the cell is committed. + # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S + # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was + # dropped even after it looked filled in on screen. + # + # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is + # not a candidate — it is somebody's row. It cannot collide with anything (there is + # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A + # merge pass that deletes what it cannot classify is not a merge, it is a filter. + keepers[rid] = r + continue + groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h), + []).append((rid, r)) + out, merged = dict(keepers), 0 + for _key, members in groups.items(): + if len(members) == 1: + out[members[0][0]] = members[0][1] + continue + merged += len(members) - 1 + # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement. + ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True) + keep_id = sorted(rid for rid, _r in members)[0] + acc = {} + for _rid, r in ordered: + for k, v in r.items(): + if k not in acc and str(v or "").strip(): + acc[k] = v + firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")] + lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")] + if firsts: + acc["first_found"] = min(firsts) + if lasts: + acc["last_found"] = max(lasts) + total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members) + if total: + acc["found_count"] = str(total) + out[keep_id] = acc + return out, merged + + +#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER +#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites +#: *"we already have a Field called stage… we don't need another checkbox for this"* — and +#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still +#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing +#: writes, nothing reads, and anybody can still click. +#: +#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the +#: record of who approved what before the wave, or leave them and let the next hand-made `tracked` +#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving +#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT +#: as a column nobody declared is the worse half. +TRACKED_KEY = "tracked" + + +def _retired_tracked_field(fields): + """The retired `tracked` column when it is still the MACHINE's own, else None. + + ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that + DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a + column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED, + or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as + an ordinary column and merely stops being fed by anything, which was already true. + + ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are + dropped unguarded: both are machine-derived cells with no history of anyone editing them. + `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test. + """ + f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None) + if f is None or f.get("type") not in ("checkbox", "", None): + return None + if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")): + return None + return f + + +def _guarded_drops(table, drops): + """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81). + + ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check. + `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks + `_retired_tracked_field` whether the column is still the machine's own, and the schema + reconciliation, which drops by KEY with no question asked. Guarding only the first was + decorative — the schema pass runs immediately afterwards on the same table and deleted the + retyped column the guard had just spared. Two paths to one deletion with one of them checked + is the shape where a control looks present and enforces nothing + ([[defects-that-mask-each-other]]). + """ + drops = set(drops or ()) + if TRACKED_KEY in drops: + stored = [f for f in (table or {}).get("fields") or [] + if str(f.get("key") or "") == TRACKED_KEY] + if stored and _retired_tracked_field(stored) is None: + drops.discard(TRACKED_KEY) + return drops + + +def _orphan_tracked_rows(fields, rows): + """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject. + + ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE + BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose + declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every + surface that renders from `fields` — and a migration that only looks at `fields` finds nothing + to do and `continue`s past the table. Storage, export and any future re-declaration of the key + all still see them. + """ + if any(str(f.get("key") or "") == TRACKED_KEY for f in fields): + return [] # the column above owns its own cells + return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})] + + +def backfill_corpus_snapshots(rt, log=print): + """The BACKFILL half: every profile row holding a measurement with no series behind it gets + its corpus observation. Returns how many were written. + + ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows + that arrived before the forward fix existed. A second row-shape here would be two ideas of + what a corpus observation is, and they would disagree on the next promoted column. + + ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is + deliberate. A profile that HAS been enriched already owns real observations; synthesising a + corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and + replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind + it, which is the only case where a corpus observation can only improve matters. + + ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on + every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has + a series, so it stops matching and the scan costs a dict comparison. + """ + snaps = ut_get(rt, IG_SNAPSHOTS_TABLE) + if snaps is None: + return 0 # no series store — see `append_ig_snapshots` on why we never mint one + known = {str((r or {}).get("influencer_key") or "").strip().lower() + for r in (snaps.get("rows") or {}).values()} + known.discard("") + #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived + #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a + #: named list goes silently empty the day the schema moves). + measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS + if fd["key"] not in _SNAPSHOT_OWN_KEYS + and fd.get("type") in ("int", "pct")) + incoming = [] + for table_key in _ig_profile_tables(rt): + for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): + handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if not handle or handle in known: + continue + if not any(str((row or {}).get(k) or "").strip() for k in measured): + continue # nothing was ever read about this row — there is no observation to record + built = corpus_snapshot_row(row) + if built: + incoming.append(built) + known.add(handle) # two profile tables holding one handle write ONE observation + return append_ig_snapshots(rt, incoming, log=log) + + +def backfill_post_snapshot_grain(rt, log=print): + """Copy `posted_at`/`type` off the POST row onto every measurement missing them. + + ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's + is not, so a pass that has already run matches nothing and a post that genuinely has no date + is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration + around it. + + ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN + THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own + record of the moment is the one to keep — a fact table that lets a dimension rewrite its + history is not a fact table. + """ + psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE) + posts = ut_get(rt, IG_POSTS_TABLE) + if psnaps is None or posts is None: + return 0 + have = {f.get("key") for f in (psnaps.get("fields") or [])} + carried = [k for k in ("posted_at", "type") if k in have] + if not carried: + return 0 # the columns are not declared yet — `ut_ensure` adds them first + by_shortcode = {} + for row in (posts.get("rows") or {}).values(): + code = str((row or {}).get("shortcode") or "").strip() + if code: + by_shortcode[code] = row or {} + changes = {} + for rid, row in (psnaps.get("rows") or {}).items(): + post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip()) + if not post: + continue + for key in carried: + want = str(post.get(key) or "").strip() + if want and not str((row or {}).get(key) or "").strip(): + changes.setdefault(str(rid), {})[key] = _s(want, 200) + if not changes: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + table = cur.get(IG_POST_SNAPSHOTS_TABLE) + if table is not None: + for rid, values in changes.items(): + table.setdefault("rows", {}).setdefault(rid, {}).update(values) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind") + return len(changes) + + +#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work +#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS. +#: +#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same +#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset +#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already +#: relational (the 12 links and rollups), or have no observation behind them at all +#: (`location_guess`/`location_confidence`, which WE infer rather than read). +#: +#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST. +#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells — +#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those +#: values were never recorded as observations (the hole this same wave closed going forward, in its +#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three +#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a +#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author +#: assumed. +#: +#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that +#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`. +#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not +#: safe today is not refused forever, it is simply not converted yet. +IG_DERIVABLE_KEYS = tuple( + fd["key"] for fd in PRESET_PROFILE_FIELDS + if fd.get("type") not in ("link", "rollup") + and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found", + "found_count", "created_by") + and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS}) + + +def _derived_profile_bag(key): + """The rollup that replaces the mapper's write of ONE profile column. + + ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's + value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an + expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the + scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest + observation that ACTUALLY READ this field", which is what the materialised cell meant. + MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the + observation itself, which is why the guard below is a proof and not a hope. + """ + return {"link": "profile_snapshots_link", "field": key, "fn": "latest", + "sortBy": "pulled_at", "sortDir": "desc", + "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"} + + +def _derived_profile_columns(table, snapshots): + """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table. + + = the ones already converted, PLUS the ones whose derived value equals the stored value on + every row. Returns keys. + + ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with + a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns — + `rollup` included, by design — so a converted column that this function stopped naming would be + re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written + by the mapper. The edit would look applied and undo itself later, which is exactly + [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore + included unconditionally rather than re-proved: its cells ARE the derived values, so a proof + would be comparing the fold against itself. + + ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody + measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere + else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest + authority is the data in front of the migration. + ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an + oversight: there is nothing to lose, and leaving it materialised would mean the schema differs + between two tenants for no reason a reader could discover. + """ + fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []} + rows = (table or {}).get("rows") or {} + by_subject = {} + for snap in ((snapshots or {}).get("rows") or {}).values(): + key = str((snap or {}).get("influencer_key") or "").strip().lower() + if key: + by_subject.setdefault(key, []).append(snap or {}) + # The engine's own ordering, once per subject — newest first, unrankable rows last. + ordered = {} + for subject, observations in by_subject.items(): + keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations] + rankable = [(o, k) for o, k in keyed if k is not None] + rankable.sort(key=lambda ok: ok[1], reverse=True) + ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None] + # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph + # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose + # link resolves to nothing — a blank cell wearing a configured column's clothes. + if "profile_snapshots_link" not in fields: + return [] + out = [] + for key in IG_DERIVABLE_KEYS: + field = fields.get(key) + if not field: + continue + if field.get("type") == "rollup": + out.append(key) # already converted — see the note above on why + continue + declared = str(field.get("type") or "text") + safe = True + for row in rows.values(): + stored = str((row or {}).get(key) or "").strip() + window = [o for o in ordered.get( + str((row or {}).get("handle") or "").strip().lower(), []) + if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here + derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared) + or "").strip() + if stored != derived: + safe = False + break + if safe: + out.append(key) + return out + + +def _as_derived_field(field): + """One preset scalar declaration → the same column declared as a rollup over the series.""" + key = str(field.get("key") or "") + return {**{k: v for k, v in field.items() if k not in ("agg",)}, + "type": "rollup", "rollup": _derived_profile_bag(key)} + + +def migrate_ig_tables(rt, log=print, only=""): + """Bring every Instagram Profile and canonical child table onto one current contract. + + Returns counted changes rather than a reassuring line. Profile cells are converted before + their type declarations change; then the graph reconciliation adds any missing preset + Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes + only retired machine fields. + + `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides + the WRITE PATH, so a table is brought forward immediately before anything appends to it and no + caller has to remember to run anything. + """ + stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0, + # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from + # `stamped` because they answer different questions: that one is "how many ROWS got a + # platform", these are "how many TABLES changed shape". + "pinned": 0, "flagged": 0, "droppedName": 0, + "droppedRecentPosts": 0, "droppedPostHashtags": 0, + "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0} + want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS} + # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red + # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a + # first run the table does not exist yet: with the detection bypassed it fell through to the + # writer and stamped a stub table with empty fields, clobbering the creation that was the whole + # point of the call (the owner stamp and the flow tag went with it). A table that does not + # exist, or that is not an Instagram profile table, has nothing to migrate. + # ⭐ WAVE 32 · T42 — the CONTRACT set, not the detector's. This loop retypes columns, stamps + # `platform` and drops retired keys, i.e. it edits the schema — so it answers to the owner's + # rule about which databases Instagram may write into, exactly like `_ig_schema_contract` at + # the bottom of this function. `_ig_profile_tables` stays the wider question and keeps its own + # callers (`discover_default_table`, `backfill_corpus_snapshots`, the detector's NC). + detected = set(_ig_contract_tables(rt)) + targets = ([only] if only in detected else []) if only else sorted(detected) + # ⭐⭐ WAVE 31 · T30 — `only` NOW NARROWS THE WHOLE FUNCTION, AND IT DID NOT. + # + # ⛔ THE BUG WAS THAT `only` NARROWED THE LOOP ABOVE AND NOTHING ELSE. The tenant-wide passes + # at the bottom — two full `_ig_schema_contract` + `_reconcile_ig_graph_fields` rounds and two + # backfills — ignore `only` entirely, so `migrate_ig_tables(only=X)` paid a whole-tenant + # Instagram reconciliation no matter what X was. And `ut_ensure` calls it that way on EVERY + # ensure whose field set intersects `PRESET_PROFILE_KEYS` — which TikTok's profile schema does, + # because the two networks share column names. + # MEASURED on the wave-30 fixture, `only="ut_tt_profile"`: `_ig_profile_tables` returns `[]`, + # `stats` comes back completely empty and ZERO store writes happen — at a cost of **10 + # whole-document reads**, each a `Store.get` deep copy (documented ceiling 35.8 MB / ~1.4 s). + # Across one save's four `ut_ensure` calls that was **40 of the 41** reads behind owner item 7's + # *"it just say Saving... and takes a long time"*. The migration was not slow; it was running at + # all. + # ⚠ THE COMMENT ONE SCREEN DOWN IS WHY THIS WENT UNSEEN FOR THREE WAVES: *"Both are guarded on + # 'is there anything to do' and cost a dict scan in the steady state, which is what makes them + # safe on a function that rides every `ut_ensure`."* That guard is on the WRITE. The read was + # never guarded, and a scan of a freshly deep-copied 35 MB document is not a dict scan + # [[read-a-gates-predicate-for-what-it-excludes]]. + # ⛔ SCOPED TO THE PROVEN-EMPTY CASE ONLY, deliberately. When `only` IS a detected profile table + # the function is unchanged, tenant-wide passes included — that is the Instagram path every + # migration check in `verify_automation` exercises. And a caller passing no `only` still gets + # the full sweep. The claim being made here is narrow and checkable: *a table that is not in the + # Instagram graph has no Instagram migration*, which is the same thing the `detected` test above + # already decided one line earlier and then threw away. + # ⛔ AND IT IS NOT A LEND. Lending one snapshot through the rest of this function would be the + # obvious next fix and it is WRONG: the second contract pass exists precisely because it must + # read what the first pass WROTE (see its own note below), so a read-write-read sequence cannot + # answer out of one snapshot without silently un-fixing that idempotency. + if only and not targets: + return stats + for table_key in targets: + tbl = ut_get(rt, table_key) or {} + fields = [dict(f) for f in tbl.get("fields") or []] + rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} + # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, + # so the cell conversions below can never run twice on the same value. + stale = [f for f in fields + if f.get("key") in want and f.get("type") != want[f["key"]] + and f.get("type") in ("text", "", None)] + stale_keys = {f["key"] for f in stale} + has_platform = any(f.get("key") == "platform" for f in fields) + needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] + # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. + # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the + # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in + # production needs them stamped here, and that is the whole reason this branch exists. + # + # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, + # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which + # is exactly why these are safe to run on every write where a value conversion would not be. + pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ + if PRESET_PINNED_KEY else None + flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ + if PRESET_FLAG_KEY else None + # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write + # doors, and a migration is not exempt from it. If somebody has already flagged another + # column on this table, stamping the handle too would leave the table in a state its own + # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would + # silently bind to whichever came first in the field list. Leave it alone and pin only. + other_profile = next((f for f in fields if f is not flag_f + and isinstance(f.get("profile"), dict)), None) + want_pin = pin_f is not None and pin_f.get("pinned") is not True + want_flag = (flag_f is not None and other_profile is None + and not isinstance(flag_f.get("profile"), dict)) + vestigial = _vestigial_name_field(fields, rows) + retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} + if _retired_tracked_field(fields) is not None: + retired.add(TRACKED_KEY) + orphan_tracked = _orphan_tracked_rows(fields, rows) + if (not stale_keys and has_platform and not needs_stamp + and not want_pin and not want_flag and vestigial is None and not retired + and not orphan_tracked): + continue + stats["tables"] += 1 + if want_pin: + pin_f["pinned"] = True + stats["pinned"] += 1 + if want_flag: + flag_f["profile"] = {"source": PROFILE_SOURCE_IG} + stats["flagged"] += 1 + if vestigial is not None: + # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer + # exists is invisible everywhere except the next export, where it reappears as a column + # nobody declared. `_vestigial_name_field` has already proven every one of them blank. + fields = [f for f in fields if f is not vestigial] + for r in rows.values(): + r.pop("name", None) + stats["droppedName"] += 1 + if retired: + fields = [f for f in fields if f.get("key") not in retired] + for r in rows.values(): + for key in retired: + r.pop(key, None) + stats["droppedRecentPosts"] += int("posts" in retired) + stats["droppedPostHashtags"] += int("post_hashtags" in retired) + stats["droppedTracked"] += int(TRACKED_KEY in retired) + for rid in orphan_tracked: + # D-74: a tick whose column was already gone. Counted apart from the column drop + # because they answer different questions — "how many tables still declared it" and + # "how many rows were still carrying it after the declaration went". + rows[rid].pop(TRACKED_KEY, None) + stats["droppedTrackedCells"] += 1 + + for key in stale_keys: + conv = _MIGRATE_CONVERT.get(key) + if not conv: + continue # int/checkbox already store the right text shape + for r in rows.values(): + if str(r.get(key) or "").strip(): + new = conv(r[key]) + # "" from a converter means UNREADABLE, and the cell keeps its original text. + # A migration that empties a cell is indistinguishable from one that moved it. + if new: + r[key], stats["converted"] = new, stats["converted"] + 1 + for f in stale: + f["type"] = want[f["key"]] + stats["retyped"] += 1 + for rid in needs_stamp: + rows[rid]["platform"] = PLATFORM_INSTAGRAM + stats["stamped"] += 1 + if not has_platform: + fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields + rows, merged = _merge_candidates(rows) + stats["merged"] += merged + + def _up(cur, _f=fields, _r=rows): + cur = cur or {} + t = dict(cur.get(table_key) or {}) + t["fields"], t["rows"] = _f, _r + cur[table_key] = t + return cur + + rt.update(UT_STORE_KEY, _up) + # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION. + # + # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early + # (`if only and not targets` above). What it left is the case the row is actually about — `only` + # naming a table that IS in the graph, i.e. **every save an Instagram tenant makes**. The four + # passes below are tenant-wide by construction: two `_ig_schema_contract` + + # `_reconcile_ig_graph_fields` rounds and two backfills, each walking the whole document, none + # of them reading `only`. So one `ut_ensure` on one table paid a whole-tenant Instagram + # reconciliation, and `ut_ensure` runs several times per save. + # + # ⛔ THE ROW'S EXIT OFFERS TWO BRANCHES AND THIS TAKES THE FIRST ONE — *"the tenant-wide passes + # are reachable from a scheduled/boot caller"* — because they ARE, and were before this change: + # `routes_tables.py:882` (`_ig_forward`) calls `migrate_ig_tables(rt)` with NO `only`, once per + # tenant per process, guarded by `_IG_FORWARDED`. That is the right home for a whole-tenant + # repair: once, off the save path, rather than on every ensure. + # ⚠ THE SECOND BRANCH — "bound them by an anything-to-do test guarded on the READ" — is NOT + # available here and the comment above says why: the second contract pass exists to read what + # the first pass WROTE, so it cannot be answered out of one snapshot without silently + # un-fixing the idempotency everything else depends on. + # ⛔ NARROW AND CHECKABLE, like its W31-T30 sibling: a call that NAMES ONE TABLE does that + # table's migration. A call that names none still does the whole sweep, unchanged, and every + # tenant-wide check in `verify_automation` goes through that path. + if only: + return stats + # The profile loop owns value conversions and deduplication. The schema pass owns the + # declarations across ALL related databases and is safe only after those conversions, because + # the stored old type is the idempotency key for unit-changing values such as engagement. + wanted, drops = _ig_schema_contract(rt) + stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops)) + # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT + # IDEMPOTENT, which is the one property everything else here is built on. + # + # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the + # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs + # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our + # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired + # profile cell against an UNREPAIRED observation, finds them different, and declines to convert + # a column that is in fact perfectly derivable. The next call then converts it — and + # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that + # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1. + # + # ⚠ The module already knew this shape one level up — the profile loop's own note says the + # schema pass "is safe only after those conversions, because the stored old type is the + # idempotency key". Same argument, one table further down. + # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same + # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is + # what moves the data, and it has already run. + wanted2, drops2 = _ig_schema_contract(rt) + stats["schema"] = int(bool(stats["schema"] + | int(_reconcile_ig_graph_fields(rt, wanted2, drops2)))) + # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose: + # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running + # it first would find nothing to fill and report a truthful zero about the wrong moment. + # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which + # is what makes them safe on a function that rides every `ut_ensure`. + stats["series"] = backfill_corpus_snapshots(rt, log=log) + stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log) + if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]: + log(f"[ig-migrate] {stats}") + return stats + + +# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ────────────────── +# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their +# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never +# hidden (R5). +# +# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN: +# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the +# corpus barely populates produces a filter that returns nothing and looks exactly like +# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about). +# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The +# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`, +# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them +# narrows under its own `default_operator`, asserted in the gate rather than assumed. +# +# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`, +# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and +# nothing else — a search that returns what you gave it, at full price. So the ranker reads only +# the three that describe a KIND of account: shared words in the bio, a shared domain in the link, +# and shared words in the name. +SEED_SOURCES = ("view", "cohort") +SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N] +SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter +SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared +SEED_MIN_ROWS = 2 # one row has nothing to share WITH +#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two +#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every +#: bio word in a two-row seed qualified as "shared", and the surface would have offered a +#: characteristic derived from a single record as the thing those records have in common. A +#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is +#: the fabrication this module refuses everywhere else. +SEED_MIN_ROWS_SHARING = 2 +#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor +#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down. +SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"), + ("full_name", "full_name")) +#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT — +#: an aggressive list would silently drop a real signal, and the coverage bar already removes most +#: noise. Anything ≤2 characters is dropped by length instead of by enumeration. +SEED_STOPWORDS = frozenset({ + "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from", + "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more", + "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"}) + + +def _seed_tokens(text): + """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars.""" + words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower()) + return {w for w in words if w not in SEED_STOPWORDS} + + +def _seed_domain(url): + """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a + domain and that IS the shared characteristic; the path is the thing that differs.""" + raw = str(url or "").strip() + if not raw: + return "" + host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower() + return host[4:] if host.startswith("www.") else host + + +def seed_predicates(rows): + """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network. + + `basis` is what makes the offer honest rather than magic: how many rows were read, which + characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads + as weak instead of being presented with the same confidence as a strong one. + + ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED + (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24 + default-versus-guard defect rebuilt — and it would be worse here, because the user did not + type it: they would be told their own product's suggestion is invalid. + + ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage + bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks + plausible, returns zero rows, and reads as "no such accounts exist". + """ + rows = [r for r in (rows or []) if isinstance(r, dict)] + total = len(rows) + basis = {"rows": total, "fields": [], + # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature. + # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1 + # preset set, so on our own seed rows it is usually simply absent — which the surface + # must SAY, or a lane that adds nothing reads as a lane that failed. + "related": {"tried": total, + "found": sum(1 for r in rows + if str(r.get("related_accounts") or "").strip())}, + "note": ""} + if total < SEED_MIN_ROWS: + basis["note"] = ("a seed needs at least two records. One record has nothing to share " + "with anything") + return [], basis + ranked = [] + for row_key, vendor_field in SEED_FROM_ROW: + counts = {} + for r in rows: + vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url" + else _seed_tokens(r.get(row_key))) + for v in vals: + if v: + counts[v] = counts.get(v, 0) + 1 + for value, n in counts.items(): + cov = n / total + if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING: + ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3), + "rows": n}) + # Strongest first; ties broken by the LONGER value, which is the more specific one. + ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"])) + derived, seen_fields = [], set() + for f in ranked: + if len(derived) >= SEED_MAX_PREDICATES: + break + # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the + # accounts carrying both words, which is a much smaller search than the seed implies. + if f["name"] in seen_fields: + continue + p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]} + if not predicate_narrows(p): + continue # cannot be offered; it would be refused at the save door + seen_fields.add(f["name"]) + derived.append(p) + basis["fields"].append({"name": f["name"], "label": field_label(f["name"]), + "value": f["value"], "coverage": f["coverage"]}) + if derived and narrowing_refusal(derived, "and"): + # Belt AND braces: the per-predicate check above should make this unreachable, and it is + # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest + # empty rather than offering something the save door will reject. + basis["note"] = ("the shared characteristics found are not specific enough to search on " + "add a condition of your own") + return [], basis + if not derived: + basis["note"] = basis["note"] or ( + "these records share no bio words, link domain or name in common. Nothing could be " + "derived, so the conditions are yours to write") + return derived, basis + + +def seed_rows(rt, table_key, view_id=""): + """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by + `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]]).""" + t = ut_get(rt, str(table_key or "")) + if t is None: + return [], f"{table_key!r} is not a database in this workspace" + rows = list((t.get("rows") or {}).values()) + if str(view_id or "").strip(): + # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a + # different row set from the view it names would be describing a view nobody has. + tree, _fields, problem = view_filter(rt, table_key, view_id) + if problem: + return [], problem + rows = [r for r in rows if lane_match(tree, r)] + return rows[:SEED_MAX_ROWS], "" + + +def automation_tables(defn): + """Every `ut_*` table THIS automation writes into — its config target plus every + `create_record` action's table, deduped, order preserved. + + ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition. + Split out rather than parameterised because the two questions are different: that one asks + "where could a Category value be, anywhere in this tenant" and is deliberately generous; this + one asks "which rows has THIS automation already found" and must not reach into a table it + does not write, or a nightly search would start excluding another automation's finds. + """ + keys, seen = [], set() + cfg = (defn or {}).get("config") or {} + for raw in [cfg.get("targetTable"), + *(((a or {}).get("config") or {}).get("table") + for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]: + k = str(raw or "").strip() + if k.startswith(UT_PREFIX) and k not in seen: + seen.add(k) + keys.append(k) + return keys + + +def candidate_key(platform, handle): + """The discovery upsert identity: `(platform, handle)`, as one string. + + ⭐ WAVE 29 — LIFTED OUT OF `run_discover_instagram`, where it was a closure, because a second + runner now needs the same identity. Two closures computing "the same" key is how one of them + grows a different default for a blank `platform` and the two networks quietly start sharing + rows — which is the exact data-loss shape wave 26's R4 added `platform` to prevent. + + ⚠ A BLANK PLATFORM DEFAULTS TO INSTAGRAM, and that is a FACT about the stored data rather than + an assumption: every row written before wave 26 came from the Instagram runner. Blank-keyed + rows would fail to match their own re-find and duplicate the whole table on the next run. + """ + return f"{str(platform or PLATFORM_INSTAGRAM).strip()}\n{str(handle or '').strip()}" + + +def already_found_handles(rt, defn, cap=None): + """The handles this automation has already written, for the vendor-side exclusion (item 7). + + ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a + profile has always been harmless — and never free: the vendor bills for every record it + returns, so a nightly search over stable keywords pays again, in full, for the same accounts + every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty + scout re-found its whole result set on every run. This is the list that stops it, sent as one + `not_in` the vendor evaluates BEFORE billing. + + ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would + hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords + are two questions, and one must not answer with "somebody already looked at that". + """ + cap = BD_EXCLUDE_MAX if cap is None else int(cap) + out, seen = [], set() + for table_key in automation_tables(defn): + for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): + h = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if h and h not in seen: + seen.add(h) + out.append(h) + if len(out) >= cap: + return out + return out + + +def observed_category_tables(rt): + """Every table a Category value could have been written into, DERIVED from the automations + this tenant actually has — the default discovery table and the snapshot table are the FLOOR, + not the list. + + ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared: + `config.targetTable` (the discovery runner's own write) and any `create_record` action's + `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker + writes). Both are read, deduped, order preserved so the defaults come first. + """ + keys, seen = [], set() + + def _add(k): + k = str(k or "").strip() + if k and k.startswith("ut_") and k not in seen: + seen.add(k) + keys.append(k) + + _add(DISCOVER_TABLE) + _add("ut_ig_snapshots") + try: + for defn in (all_definitions(rt) or {}).values(): + cfg = (defn or {}).get("config") or {} + _add(cfg.get("targetTable")) + for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")): + _add(((act or {}).get("config") or {}).get("table")) + except Exception: # noqa: BLE001 + # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults — + # the same posture as the master lane below: degrade to less vocabulary, never to an error. + pass + return keys + + +def observed_categories(rt, limit=60): + """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts + free text. + + D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen + (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that + still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**: + transcribe a published taxonomy and hope it lines up."* This is the first branch. + + ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the + corpus does not use returns zero rows and looks exactly like an honest "no such accounts + exist" — so a picker built from Instagram's own category list would mislead precisely when it + looked most authoritative. Every option here has been observed on a real row, and each carries + its COUNT so a value seen once reads differently from one seen forty times. + + ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a + control that refused anything else would be a second, quieter version of the same lie. + """ + seen = {} + + def _eat(rows, key): + for r in rows or []: + v = " ".join(str((r or {}).get(key) or "").split())[:60] + if v: + seen[v] = seen.get(v, 0) + 1 + + # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped + # green. The two constants below were the whole list, which was correct until **wave 25 R2 + # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a + # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely + # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows + # while the tenant's two real target tables held 20 each, so this function honestly reported + # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that + # names its sources goes quietly empty the moment the product lets a user choose one. + for key in observed_category_tables(rt): + _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category") + try: + # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three + # candidates still gets a vocabulary drawn from every profile the platform has captured. + import ig_master + if ig_master.configured(): + # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which + # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers + # None, so the master lane would have degraded to nothing INSIDE the except below and + # this would have shipped as a silent no-op with the gate green. Verified against + # `ig_master.series_for`, which reads the same bucket the same way. + _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category") + except Exception: # noqa: BLE001 + # A master that cannot be read costs the tenant its own values and nothing else. + pass + return [{"value": v, "count": n} + for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]] + + +def preset_plan(rt, table_key): + """C1 / R2b: the preset set diffed against ONE database's CURRENT fields. + + `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` + + ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns + an automation will USE and which it will CREATE — and the only thing that can answer it is + whatever holds both lists. A client that diffed the preset set against a table's fields would + be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that + actually runs; it would be wrong precisely when the two lists differ, which is the only case + anybody is asking about. + + ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save + moment, and the honest answer is "all of them will be created". `present` is a fact about the + table, so it reads False for every field rather than the call refusing. + + ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG + (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the + relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY. + That is CORRECT for this payload — it answers "which columns will be used vs created", where + a name and a type are the whole question — and it is safe today because nothing RENDERS a + column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes + `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from + `PRESET_PROFILE_FIELDS`, which carries the bag. + ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would + render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or + widen this projection deliberately; do not assume a bag is here because the field has one. + """ + have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields") + or [])} + fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text", + "present": f["key"] in have} + for f in PRESET_PROFILE_FIELDS] + return {"table": str(table_key or ""), + "exists": ut_get(rt, str(table_key or "")) is not None, + "fields": fields, + "willUse": [f for f in fields if f["present"]], + "willCreate": [f for f in fields if not f["present"]]} + + +def clean_predicates(raw, operator="and", kind=""): + """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. + + Refuses rather than coerces: a predicate naming a field the API rejects comes back as a + 400 with the field named, because the alternative — dropping it — turns "find me verified + accounts in Georgia" into "find me any account" and bills for the difference. + + ⭐ WAVE 32 · T46 (D-167) — `kind` NARROWS THE VOCABULARY TO THE PLATFORM'S. Default `""` keeps + Instagram's 21 names, so every existing caller and every stored Instagram automation is + unchanged; a `discover_tiktok` is validated against the 5 names its corpus actually has. The + refusal it produces is the same sentence, listing the platform's own fields — which is the + difference between a search that says why it cannot be built and one that is built, paid for, + and comes back empty. + """ + fields, _lead = filter_fields(kind) + out = [] + for p in (raw or [])[:12]: + if not isinstance(p, dict): + continue + name = _s(p.get("name") or p.get("field"), 60).strip() + op = _s(p.get("operator") or p.get("op"), 20).strip() + if name not in fields: + return None, ("that field cannot be searched. The searchable ones are: " + + ", ".join(field_label(f) for f in fields)) + # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly + # because every operator was legal on every field; a yes/no column offering "at least" + # is a question with no meaning that the vendor is nevertheless asked. + allowed = ops_for(name) + if op not in allowed: + return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r}. " + "it takes: " + + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) + entry = {"name": name, "operator": op} + if op not in BD_NULLARY_OPS: + raw_v = p.get("value") + # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and + # it is stored as a list rather than as N sibling predicates so the row a person sees + # and the row that is stored are the same thing. + vals = raw_v if isinstance(raw_v, list) else [raw_v] + vals = [v.strip() if isinstance(v, str) else v for v in vals] + vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))] + if not vals: + return None, f"give {field_label(name)} something to compare against" + if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: + return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value. " + f"{field_label(name)} can only be given a list with " + + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) + if o in BD_MULTI_VALUE_OPS)) + if len(vals) > MAX_PREDICATE_VALUES: + return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values " + "in one condition") + if field_kind(name) == "boolean": + ok = {o["value"] for o in BD_BOOLEAN_OPTIONS} + bad = [v for v in vals if str(v).lower() not in ok] + if bad: + return None, f"{field_label(name)} is answered Yes or No" + # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers + # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and + # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field + # was a free-text box until this wave, so the Yes/No dropdown that makes it easy + # to ask is also the thing that would have made every boolean condition fail. + # The dropdown's option VALUES stay "true"/"false" (a carries strings); - # the conversion belongs here, at the edge that talks to the vendor. - vals = [str(v).lower() == "true" for v in vals] - if field_kind(name) == "number": - try: - vals = [float(v) if "." in str(v) else int(v) for v in vals] - except (TypeError, ValueError): - return None, f"{field_label(name)} takes a number" - # A single value stays a SCALAR on the wire. The vendor has only ever been sent - # scalars for these operators; the list form is expanded at send time and the - # one-value case must not quietly start exercising an untested shape. - entry["value"] = vals[0] if len(vals) == 1 else vals - out.append(entry) - if not out: - # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES. - # This used to refuse, which was right while a wizard collected the filters before the - # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger - # CREATES the automation, and the filters are typed afterwards — so a save-time refusal - # here meant the trigger could never be picked at all. It is the A3 - # stored-inert-with-`configured:false` pattern this module already runs on. - # - # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient: - # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a - # search, deliberately, because "a stored config can predate the law, and the vendor - # bills for breadth whether the filter was saved yesterday or last month". The RUN is the - # wall. Every MALFORMED predicate above is still refused here, and the narrowing law - # below still refuses a non-empty filter that narrows nothing — only EMPTY changed. - return [], None - # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the - # measured hang this guards against. Same sentence at create/patch and at run. - guard_err = narrowing_refusal(out, operator, kind) or depth_refusal(out, operator) - if guard_err: - return None, guard_err - return out, None - - -def discover_estimate(records_limit): - """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the - section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" - n = max(0, int(records_limit or 0)) - return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), - "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", - "note": "estimated from the published rate. An exact price is only known after a run" - "a price before a run, and this account's token cannot read a balance"} - - -def _candidate_row(row, stamp): - """One corpus row → a `ut_ig_candidates` row. None when it has no handle. - - ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already - stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted - `tracked` either; R6 deleted that column in wave 26.) - """ - if not isinstance(row, dict): - return None - handle = str(_first(row, "account", "username", default="") or "").strip() - if not handle: - return None - return { - # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads - # Instagram, so it is a constant here rather than something derived from the row; the - # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart. - "platform": PLATFORM_INSTAGRAM, - "handle": handle, - "profile_url": str(_first(row, "profile_url", "url", - default=f"https://www.instagram.com/{handle}/")), - "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), - "followers": _s(_ig_int(_first(row, "followers"))), - "following": _s(_ig_int(_first(row, "following"))), - # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is - # the path where it carries values (measured on all five of the 2026-08-04 result set). - # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`. - "avg_engagement": _pct100(_first(row, "avg_engagement", default="")), - "bio": _s(_first(row, "biography", "bio", default=""), 500), - "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), - "verified": "1" if _first(row, "is_verified", default=False) else "", - "category": _s(_first(row, "category_name", "business_category_name", default="")), - # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp - # still exists on the snapshot series, which is where a time axis belongs. - "last_found": _day(stamp), - } - - -# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ────────────────────────────────────────── -# -# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a -# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a -# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via -# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers, -# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at -# all, so discovery was the one rung in this module that read numbers and recorded no observation. -# -# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold -# real corpus numbers that are already on screen and already filterable. The gap is that the -# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law -# exists to forbid one level up. -# -# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make -# it honest already existed: `source` says **Discovery** (this was read off the vendor's -# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not -# read this as an exact count taken at `pulled_at`" — which is the whole difference between -# recording what we know and inventing what we do not. -# -# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it -# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as -# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over -# the series therefore prefers the exact read over the corpus read whenever both happened on one -# day — the ordering you would have to hand-write, falling out of the grain. - -#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell. -IG_VIA_DISCOVERY = "brightdata:discovery" - -#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement -#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather -#: than typed out — a hand-list is what silently stops carrying the next promoted column. -_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") - - -def corpus_snapshot_row(row, day=""): - """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable. - - `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's - only date is `first_found`. - - ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is - `@`, so re-running this over the same rows on the same day rewrites the same key - and `upsert_rows` merges it — the migration is safe to call on every write, which is the only - way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys - idempotency on the stored TYPE). - - ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty - ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed. - Here the merge is the hazard instead: a second discovery on the same day returning a thinner - corpus record would otherwise ERASE what the first one learned. An absent key renders exactly - as an empty cell, so nothing is lost by leaving it out. - """ - handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if not handle: - return None - when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "") - if not when: - return None - out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when, - "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"} - for fd in SNAPSHOT_FIELDS: - key = fd["key"] - if key in _SNAPSHOT_OWN_KEYS: - continue - value = (row or {}).get(key) - if str(value or "").strip(): - out[key] = str(value) if key == "source_payload" else _s(value, 400) - return out - - -def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print): - """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written. - - ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only - honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls - `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the - creator from there is unbounded recursion. A tenant with no series store has nothing to - back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this - function then has somewhere to write. - """ - rows_in = [r for r in (incoming or []) if r] - if not rows_in: - return 0 - existing = ut_get(rt, snap_key) - if existing is None: - return 0 - merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key", - cap=row_cap(snap_key)) - if counts["capped"]: - # D-11's law: a full append table means the SERIES has stopped growing, which is the one - # failure a chart cannot show you. - log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by " - f"{snap_key}'s row cap") - written = counts["inserted"] + counts["updated"] - if not written: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if cur.get(snap_key) is not None: - cur[snap_key]["rows"] = merged - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return written - - -# --------------------------------------------------------------------------------------------- -# RUNNERS -# --------------------------------------------------------------------------------------------- - -def cap_note(capped, missing=()): - """The sentence(s) a cap breach adds to a run summary. - - `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. - `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not - be created at all. - - ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the - ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that - does not say WHERE is only marginally better than a silent one. - - ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused - SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the - runners' `if tgt is not None` write guard then skipped that bucket, and the run reported - success over a database that does not exist. Exactly the silent-stop failure the row cap - already had, one level up — and harder to spot, because the table is not there to look at. - """ - out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows). {n} row{'' if n == 1 else 's'} " - f"from this run were NOT written" for k, n in (capped or []) if n] - out += [f"⚠ the {k} database could NOT BE CREATED. This workspace is at its " - f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" - for k in (missing or [])] - return "; ".join(out) - - -def ut_missing(rt, *keys): - """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" - have = ut_all(rt) - return [k for k in keys if k and k not in have] - - -def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" - cfg = defn.get("config") or {} - fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") - dry = bool(cfg.get("dryRun")) - # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from - # the rollup — inferring is exactly how a green dot ended up over an empty table (see the - # rollup comment in `run_field_instagram`). - steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} - step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}") - status, final, body = fetch(cfg["url"]) - if not (200 <= status < 300): - steps["fetch"] = "error" - return ("error", f"{cfg['url']} answered {status}. Nothing was written", - {"status": status}, [], steps) - steps["fetch"] = "ok" - soup = _soup(body) - if cfg.get("extract") == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 - raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] - if not raw_rows: - steps["extract"] = "error" - return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], - steps) - steps["extract"] = "ok" - step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}") - - incoming = [] - for r in raw_rows: - mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} - if any(v for v in mapped.values()): - incoming.append(mapped) - - fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) - for src, tgt in fmap.items()] - # the key column leads, so the table opens on the identity it is upserted by - fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) - label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" - if dry: - # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, - # which would create the table: a dry run that leaves a new empty database behind is not - # a dry run. - table_key = ut_key_for(label, cfg.get("targetTable") or None) - else: - table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) - t = ut_get(rt, table_key) or {} - # ⛔ D-116 — THE ONE CALL SITE THAT NEEDS A PRECEDENCE RULE, because it is the one whose rows - # come from a CORPUS. Every other `upsert_rows` caller writes measurements or appends a series. - rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, - cap=row_cap(table_key), protect=corpus_protect) - touched = [rid for rid, row in rows.items() - if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] - if not dry: - ut_write_rows(rt, table_key, rows) - counts["source_rows"] = len(incoming) - affected = touched[:200] - summary = (f"{counts['inserted']} new, {counts['updated']} updated, " - f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " - f"(kept)") - # ⛔ D-116 — SAY IT. A search that quietly declines to overwrite six measured cells is doing - # the right thing invisibly, which is indistinguishable from a corpus that happened to agree — - # and the next person to wonder why a number did not move has nothing to read. The observation - # still lands in the series either way; this sentence is about the SCALAR. - if counts.get("held"): - summary += (f", {counts['held']} measured value(s) kept over the corpus " - f"(an enrichment read them exactly; the search's own numbers are in the " - f"snapshot series)") - capped = [(table_key, counts.get("capped", 0))] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - note = cap_note(capped, missing) - if note: - summary += f". {note}" - steps["write"] = "partial" - if dry: - summary = f"Test run. Nothing saved. Would have written: {summary}" - steps["write"] = "skipped" - elif not note: - steps["write"] = "ok" - state = "partial" if (counts.get("capped") or counts.get("skipped") or missing - or dry) else "ok" - log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -def capture_rows(res, pulled): - """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`. - - ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT - RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path — - do not fork them." A second copy of this mapping is the shape that goes wrong invisibly, - because the two copies would each be *plausible* and would disagree only on the rows a - particular rung happened to return. - - ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE: - * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every - time and a blank is unambiguous — it means THIS PULL did not read it. - * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are - readable on some rungs and not others (the Bright Data PROFILE row carries post identity - with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and - `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run - ERASE what an expensive one learned. - * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is - the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement - rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k - ceiling and drawing a chart of nothing. A row in a time series should mean "this was true - then"; a row meaning "nobody looked" belongs nowhere. - """ - prof = (res or {}).get("profile") or {} - snap_row = { - "snapshot_key": f"{prof.get('username')}@{pulled}", - "influencer_key": prof.get("username"), "pulled_at": pulled, - # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called - # "Read via". `res["via"]` keeps the real key for the call graph and the server log. - "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")), - "bio": _s(prof.get("bio"), 500), - "external_url": _s(prof.get("external_url"), 300), - } - for fd in SNAPSHOT_FIELDS: - k = fd["key"] - if k not in snap_row: - value = prof.get(k) - # Profile history declares the same percentage dialect as the current Profile row. - # Converting only the latest projection left the append series at 0-1 while its field - # now said pct (0-100), so the two views of one measurement disagreed by 100x. - if k == "source_payload": - snap_row[k] = str(value or "") - else: - snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400) - idents, metrics_rows, comment_rows = [], [], [] - for p in (res or {}).get("posts") or []: - ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"), - "url": _s(p.get("url"), 300)} - for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), - ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), - ("alt_text", 400), ("tagged_location", 400)): - if p.get(k): - ident[k] = _s(p.get(k), n) - if p.get("source_payload"): - ident["source_payload"] = str(p["source_payload"]) - metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")} - # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF. - # - # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average - # views over the last 12 posts" would have to walk profile → posts → each post's most - # recent snapshot, and a two-hop rollup is a much larger feature with a much worse - # invalidation story. The post row now carries its own latest, exactly as the profile row - # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3). - # - # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the - # authoritative engagement series and still gets its appended row. These cells are a - # projection of the row being appended in the same breath, never a second source. - # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent - # key leaves the previous pull's value standing — which is the correct behaviour for a - # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing - # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the - # same failure the snapshot-append rule above is written against. - measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""} - if measured: - ident.update({k: _s(v, 40) for k, v in measured.items()}) - # The stamp is what makes the numbers readable: a blank `plays` beside a - # `measured_at` of last week means "we looked and the vendor had nothing", and with - # no stamp it means that AND "we never looked", indistinguishably. - ident["measured_at"] = _day(pulled) - idents.append(ident) - if any(v is not None for v in metrics.values()): - metrics_rows.append({ - "post_snapshot_key": f"{p['shortcode']}@{pulled}", - "shortcode": p["shortcode"], "influencer_key": prof.get("username"), - "pulled_at": pulled, - # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the - # SAME vendor record the identity row above is built from, so they cannot disagree - # with it on this pull. - # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split` - # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull - # that learns a post's engagement often does not learn its date in the same - # breath. `backfill_post_snapshot_grain` fills those from the post row, which by - # then may have learned it from a different rung. - "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200), - "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), - "views": _s(metrics["views"]), "plays": _s(metrics["plays"]), - "source_payload": str(p.get("source_payload") or "")}) - for comment in p.get("embedded_comments") or []: - # The embedded shape often names only the commenter. Its parent Post is the - # authoritative owner, so bind every preview to this profile and post explicitly. - comment_rows.append({**comment, "shortcode": p["shortcode"], - "influencer_key": prof.get("username") or ""}) - comment_rows.extend((res or {}).get("comments") or []) - # The same comment can be returned as both `latest_comments` and `top_comments`, or by the - # optional full endpoint after an embedded preview. The canonical key is the one row truth. - unique_comments = {} - for row in comment_rows: - key = str((row or {}).get("comment_key") or "") - if key: - unique_comments[key] = {**(unique_comments.get(key) or {}), **row} - return snap_row, idents, metrics_rows, list(unique_comments.values()) - - -#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched. -#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of -#: `row[...] = prof.get(...)` that a future field addition can silently miss. -#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did -#: not read it", and `enriched_at` is what distinguishes that from "never enriched"). -PRESET_FROM_PROFILE = { - "username": "handle", "profile_url": "profile_url", "full_name": "full_name", - "followers": "followers", "following": "following", "avg_engagement": "avg_engagement", - "bio": "bio", "external_url": "external_url", "verified": "verified", - "category": "category", "posts_count": "posts_count", - "highlights_count": "highlights_count", "is_business": "is_business", - "is_professional": "is_professional", "ig_id": "ig_id", - # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column - # that can only ever be blank, so "make the preset fields populated" is THIS half of the - # owner's instruction and the field list is only the other half. - # ⚠ The two lists are held in step by a derived gate rather than by care: every - # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared - # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`). - "business_category": "business_category", "is_private": "is_private", - "bio_hashtags": "bio_hashtags", - "pronouns": "pronouns", "profile_name": "profile_name", - "is_joined_recently": "is_joined_recently", "has_channel": "has_channel", - "partner_id": "partner_id", "external_url_title": "external_url_title", - "fbid": "fbid", "related_accounts": "related_accounts", - "country_code": "country_code", "source_payload": "source_payload", -} - -#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a -#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate -#: could only be written as "these three are fine" — a hard-coded exception list, which is the -#: shape [[gate-answers-the-wrong-question]] warns about. -#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an -#: enrichment run — which is the whole point of them: they stay true when the LINKED table -#: changes, and a pull that touched no profile still updates a profile's post count. -PRESET_WRITTEN_ELSEWHERE = ( - "platform", "handle", "enriched_at", - "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link", - "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12", - "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured", -) - - -#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a -#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING` -#: refuses on the discovery side for exactly the same reason. -LOCATION_MIN_POSTS = 2 - - -def _post_place(post): - """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload - second. - - ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the - vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the - `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details` - shape. The complete vendor row is retained in `source_payload` on the same record, already - paid for, so a post whose normalised field came back blank can still be read here. - ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region, - country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to - "where is this person"; "Indonesia" mostly is not. - """ - if not isinstance(post, dict): - return "" - raw = str(post.get("tagged_location") or "").strip() - if not raw: - try: - payload = json.loads(post.get("source_payload") or "{}") - except Exception: # noqa: BLE001 - payload = {} - raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else "" - return raw.split(",")[0].strip() - - -def location_guess(posts): - """`(city, confidence_pct)` from the places a creator's own posts were tagged in. - - `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not - the share of ALL posts, because a creator who geotags three of twelve is telling us about - three, and dividing by twelve would report a real signal as weak. How thin the evidence is - stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two. - """ - places = [p for p in (_post_place(x) for x in posts or []) if p] - if len(places) < LOCATION_MIN_POSTS: - return "", None - counts = {} - for p in places: - counts[p] = counts.get(p, 0) + 1 - # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be - # resolved alphabetically, which is an arbitrary answer wearing a confident number. - best = max(places, key=lambda p: (counts[p], -places.index(p))) - return best, round(100.0 * counts[best] / len(places), 1) - - -def preset_cells(res, pulled): - """C4/R3: one pull → the LATEST-value cells written onto the enriched record. - - ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the - other fifteen readable: without it a blank `followers` means both "never enriched" and - "enriched in March and the vendor had nothing", and no cell on the row can tell them apart. - """ - prof = (res or {}).get("profile") or {} - cells = {} - for src, dest in PRESET_FROM_PROFILE.items(): - v = prof.get(src) - if v is not None and v != "": - # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The - # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a - # straight-through copy here would write the same wrong number the discovery path - # used to write. Both writers convert; neither is the exception. - cells[dest] = (_pct100(v) if dest == "avg_engagement" else - (str(v) if dest == "source_payload" else _s(v, 500))) - # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an - # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have - # arrived with no platform at all — and a blank half of the dedup key is how one account - # becomes two rows. - cells["platform"] = PLATFORM_INSTAGRAM - # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when - # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose - # twelve posts happened to carry no geotag would make the column worse the more it ran. - place, confidence = location_guess((res or {}).get("posts")) - if place: - cells["location_guess"] = _s(place, 120) - cells["location_confidence"] = str(confidence) - # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a - # question in days; the full stamp keeps its precision on the snapshot series. - cells["enriched_at"] = _day(pulled) - return cells - - -#: Keys the TikTok SNAPSHOT row owns itself, so the copy loop below never overwrites them from the -#: profile map. Mirrors `_SNAPSHOT_OWN_KEYS` on the Instagram side. -_TT_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") - - -def tt_preset_cells(res, pulled): - """⭐⭐ WAVE 30 · T08 — one TikTok pull → the LATEST-value cells written onto the record. - - ⛔ THIS IS NOT `preset_cells` WITH A FLAG, AND THAT IS THE WHOLE DESIGN DECISION. Instagram's - `preset_cells` walks `PRESET_FROM_PROFILE` (an Instagram map), stamps - `cells["platform"] = PLATFORM_INSTAGRAM` UNCONDITIONALLY, and computes a location guess from - posts. Passing a TikTok row through it would stamp every TikTok creator as Instagram — which is - the identity half of W26's `(platform, handle)` key, so it would not merely mislabel a row, it - would MERGE two different people's accounts under one key. - ⭐ AND THE TIKTOK ROW DOES NOT NEED A MAP AT ALL. `connectors_tt.normalize_profile` already - emits our column names — it is the SAME function the discovery runner uses — so a scraped row - and a corpus row cannot disagree about which vendor key became which column. All this adds is - what the FETCH knows and the map cannot: when it was read, and by which route. - - ⚠ FILTERED TO THE DECLARED SCHEMA. `ut_write_rows`/`_clean_field` DROP an undeclared key with - no error and a successful-looking run (the defect `section_w29_tiktok_schema`'s emit-vs-declare - sweep exists for), so an unknown key is refused here where it is visible rather than swallowed - three layers down. - """ - prof = (res or {}).get("profile") or {} - declared = {f["key"] for f in TT_PROFILE_FIELDS} - cells = {k: v for k, v in prof.items() if k in declared} - # `source` is the ROUTE that answered, not the vendor's name — the same thing `via` carries on - # the Instagram snapshot, and the reason a stored measurement can always say how it was read. - if (res or {}).get("via"): - cells["source"] = _s((res or {}).get("via"), 60) - # R3, as on the Instagram side: a `date`, because "how stale is this number" is a question in - # days. Without it a blank `followers` means both "never enriched" and "enriched and empty". - cells["enriched_at"] = _day(pulled) - return cells - - -#: The metric columns a TikTok post SNAPSHOT carries. ⛔ DERIVED from the declaration, minus the -#: keys the snapshot owns itself — so adding a metric to `TT_POST_SNAPSHOT_FIELDS` starts being -#: captured, and adding an IDENTITY column to it never does. -_TT_PSNAP_OWN_KEYS = ("platform", "post_snapshot_key", "shortcode", "influencer_key", "pulled_at", - "post_link") -TT_PSNAP_METRIC_KEYS = tuple(f["key"] for f in TT_POST_SNAPSHOT_FIELDS - if f["key"] not in _TT_PSNAP_OWN_KEYS) - - -def tt_capture_rows(res, pulled): - """⭐⭐ WAVE 30 · T10 — one TikTok pull → `(post_identity_rows, post_metric_rows, comment_rows)`. - - The TikTok twin of `capture_rows`, and separate for the same reason `tt_preset_cells` is: that - function walks Instagram's `SNAPSHOT_FIELDS`/`PRESET_FROM_PROFILE` and stamps Instagram's - platform. The mappers have already done the vendor→our-keys work here - (`connectors_tt.normalize_post` / `normalize_comment`), so this adds only what the FETCH knows. - - ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE, exactly as on the Instagram side: - * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED — `upsert_rows` writes the keys it - is handed, so sending "" would let a thin run ERASE what a full one learned. - * A METRIC SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. A row in a time series - should mean "this was true then"; a row meaning "nobody looked" eats the cap and draws a - chart of nothing. - - ⛔⛔ D-117 IS NOT REPRODUCED HERE, AND THAT IS THE INSTRUCTION THE TICKET LEADS WITH. - Instagram denormalises `posted_at` and `type` onto its post SNAPSHOT rows, where they never - reconcile against the identity table and quietly become a second, ageing answer to a question - the posts table already answers. This snapshot carries the METRICS and the join key and nothing - else — `TT_PSNAP_METRIC_KEYS` is derived from the declaration minus the keys the snapshot owns, - so the exclusion is structural rather than a list somebody has to remember to keep short. - """ - prof = (res or {}).get("profile") or {} - who = str(prof.get("handle") or "").strip().lstrip("@").lower() - idents, metric_rows, comment_rows = [], [], [] - for p in (res or {}).get("posts") or []: - shortcode = str((p or {}).get("shortcode") or "").strip() - if not shortcode: - continue - ident = {k: v for k, v in p.items() if v not in (None, "")} - # ⚠ The Posts dataset carries `profile_username`, but a row that omits it must still join: - # the profile whose `top_videos` we scraped IS the influencer, and that is a fact of the - # call rather than of the row. - ident.setdefault("influencer_key", who) - ident["platform"] = PLATFORM_TIKTOK - idents.append(ident) - measured = {k: p.get(k) for k in TT_PSNAP_METRIC_KEYS - if k != "source_payload" and p.get(k) is not None} - if not measured: - continue - row = {"platform": PLATFORM_TIKTOK, "shortcode": shortcode, - "influencer_key": str(p.get("influencer_key") or who), - "post_snapshot_key": f"{shortcode}@{pulled}", "pulled_at": pulled} - for k, v in measured.items(): - row[k] = _s(v, 400) - metric_rows.append(row) - for c in (res or {}).get("comments") or []: - if not str((c or {}).get("comment_key") or "").strip(): - continue - row = {k: v for k, v in c.items() if v not in (None, "")} - # Same fact-of-the-call argument: the Comments dataset has no influencer field at all, so - # without this the comments table could never be filtered by creator. - row.setdefault("influencer_key", who) - row["platform"] = PLATFORM_TIKTOK - comment_rows.append(row) - return idents, metric_rows, comment_rows - - -def tt_snapshot_row(res, pulled): - """One TikTok pull → its `ut_tt_snapshots` observation, or None when there is no handle. - - ⛔ THE APPEND LAW, UNCHANGED FROM INSTAGRAM AND FOR THE SAME MEASURED REASON: TikTok's Profiles - dataset carries **no measurement timestamp** (`create_time` is when the ACCOUNT was made, not - when `followers` was true — probed, `tiktok-capture.md`). So the series is dated by when WE - read it, and the key is `@` — idempotent by its KEY rather than by a flag, - so re-running over the same rows rewrites the same row instead of growing the table. - ⚠ `pulled_at` keeps the FULL stamp while the record's `enriched_at` is a day: the record answers - "how stale", the series answers "when exactly", and collapsing the second into the first would - make two reads on one day indistinguishable. - """ - prof = (res or {}).get("profile") or {} - handle = str(prof.get("handle") or "").strip().lstrip("@").lower() - if not handle: - return None - out = {"platform": PLATFORM_TIKTOK, "snapshot_key": f"{handle}@{pulled}", - "influencer_key": handle, "pulled_at": pulled, - "source": _s((res or {}).get("via") or "brightdata", 60)} - for fd in TT_SNAPSHOT_FIELDS: - key = fd["key"] - if key in _TT_SNAPSHOT_OWN_KEYS or key == "platform": - continue - value = prof.get(key) - # ⚠ A ZERO IS A MEASUREMENT AND SURVIVES — the test is `is not None`, never truthiness. - # `str(value or "").strip()` would drop a genuine 0 follower count, and this is a series - # whose whole purpose is that a number moved. - if value is not None and str(value).strip() != "": - out[key] = str(value) if key == "source_payload" else _s(value, 400) - return out - - -def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, - rows=None): - """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public - profile through the paid capability chain, write a status string into the automation column, - and append a timestamped row to each of the three IG tables. - - ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per - CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two - "which rung answered" counters this function used to keep have nothing left to distinguish.""" - cfg = defn.get("config") or {} - table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") - post_metrics = bool(cfg.get("postMetrics")) - comment_metrics = bool(cfg.get("commentMetrics")) - dry = bool(cfg.get("dryRun")) - # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a - # status written under a node id `graph()` no longer emits is a dot nothing renders, which is - # indistinguishable from a step that never ran. - steps = {"trigger": "ok", "source": "idle", - "capture_posts": "idle" if post_metrics else "skipped", - "capture_comments": "idle" if comment_metrics else "skipped", - "write": "idle"} - t = ut_get(rt, table_key) - if not t: - steps["source"] = "error" - return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) - url_field = cfg.get("urlField") or _auto_url_field(t, fkey) - if not url_field: - steps["source"] = "error" - return ("error", "the automation column has no URL field bound to it", {}, [], steps) - steps["source"] = "ok" - rows = dict(t.get("rows") or {}) - # the relational tables the pull lands in (R7): every row timestamped for time-range filters - if dry: - # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) - snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, - IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE) - else: - graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), - profile_table=table_key) - snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], - graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) - missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key) - snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) - posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) - psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) - comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {}) - - counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, - "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0, - "missing_tables": len(missing)} - cells, affected, notes = {}, [], [] - pending_metrics = [] - # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` - # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. - # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation - # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) - in_snaps, in_posts, in_psnaps, in_comments = [], [], [], [] - targets = [(rid, str(r.get(url_field, "") or "").strip()) - for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] - for i, (rid, url) in enumerate(targets): - if i: - time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) - # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile - # column automation blocks for minutes by design. A counting step is the difference - # between "working through them" and "stuck". - step(f"Capturing profile {i + 1} of {len(targets)}") - counts["profiles"] += 1 - res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log, - post_metrics=post_metrics, comment_metrics=comment_metrics, - pending_metrics=(pending_metrics if not dry else None)) - pulled = _iso() - via = res.get("via") or "" - read_ok = res["state"] in ("ok", "partial") - # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID - # vendor actually answered, which is the run's spend report — the thing the owner reads to - # reconcile a bill. What died is the free rung it used to be contrasted with, so the test - # is now simply "did a vendor answer" rather than "did the vendor we were told to try". - if read_ok and via in ("brightdata", "apify"): - counts["paid"] += 1 - if read_ok: - counts["ok" if res["state"] == "ok" else "partial"] += 1 - prof = res["profile"] - snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled) - in_snaps.append(snap_row) - in_posts.extend(ident_rows) - in_psnaps.extend(metric_rows) - in_comments.extend(captured_comments) - counts["metrics"] += len(metric_rows) - counts["comment_rows"] += len(captured_comments) - counts["posts"] += len(res.get("posts") or []) - detail = f"{prof.get('followers') or '?'} followers" - if prof.get("approx"): - detail += " (approx)" - if res.get("posts"): - detail += f", {len(res['posts'])} posts" - elif res["state"] == "partial": - detail += ", posts not readable on the rung that answered" - cells[rid] = f"{res['state']} · {_stamp()} · {detail}" - elif res["state"] == "blocked": - counts["blocked"] += 1 - cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" - notes.append(res.get("note") or "blocked") - else: - counts["error"] += 1 - cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" - notes.append(res.get("note") or "error") - affected.append(rid) - - # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. - snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) - posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at") - posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) - c_post["duplicates"] += collapsed_posts - psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) - comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key") - comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key)) - c_comments["duplicates"] += collapsed_comments - counts["new_posts"] = c_post["inserted"] - capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), - (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])] - counts["capped"] = sum(n for _k, n in capped) - - if not dry: - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for rid, val in cells.items(): - tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val - for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), - (comment_key, comments)): - tgt = cur.get(k) - if tgt is not None: - tgt["rows"] = rws - _refresh_relations_inplace(cur, log=log) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables - queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics) - if queued: - counts["metric_batches_pending"] = queued - - # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: - # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), - # `error` a LOUD partial — a pooled history with silent holes is worse than none. - master_note = "" - if not dry and (in_snaps or in_posts or in_psnaps): - import ig_master - m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), - in_snaps, in_posts, in_psnaps) - if m_status == "error": - master_note = f"⚠ the platform master copy FAILED. {m_note}; the tenant copy " \ - f"is complete and the next run re-appends" - counts["master_failed"] = 1 - elif m_status == "ok": - counts["master"] = len(in_snaps) + len(in_psnaps) - - # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while - # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail - # showed a green dot over a table with no posts in it. A summary that disagrees with the - # cells it summarises is the failure this whole module's honest-status rule exists to - # prevent, so `partial` now propagates. Measured 2026-08-04. - read = counts["ok"] + counts["partial"] - if not read: - state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" - elif counts["blocked"] or counts["error"] or counts["partial"]: - state = "partial" - else: - state = "ok" - if counts.get("metric_batches_pending"): - state = "partial" - # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) - if not targets: - cap_state = "idle" - elif not read: - cap_state = "error" if counts["error"] else "blocked" - else: - cap_state = "partial" if (counts["blocked"] or counts["error"] - or counts["partial"]) else "ok" - # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to - # read a profile now, and the node that names the profile set is the honest owner of "did - # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch - # that no longer exists in behaviour OR on screen. - steps["source"] = cap_state if targets else steps["source"] - # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually - # appended. "It was switched on" is not the same fact as "it answered", and painting the - # second from the first is the fabrication `node_status` refuses to make. - # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A - # run that found no posts to enrich did not have a rung refuse it — nothing was requested — - # so that reads `skipped`, the same word an off switch earns. - steps["capture_posts"] = ("ok" if counts["metrics"] - else "partial" if counts.get("metric_batches_pending") - else "blocked" if (post_metrics and counts["posts"]) - else "skipped") - steps["capture_comments"] = ("ok" if counts.get("comments") - else "blocked" if (comment_metrics and counts["posts"]) - else "skipped") - - summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " - f"({counts['new_posts']} new)") - if counts.get("metric_batches_pending"): - summary += (f", {counts['metric_batches_pending']} post-engagement batch" - f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building " - "(collected automatically)") - if counts["paid"]: - summary += f", {counts['paid']} with exact counts" - if counts["metrics"]: - summary += f", {counts['metrics']} post engagement snapshots" - elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"): - # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there - # was no engagement", which is a claim about Instagram rather than about our run. - summary += ", no post engagement was readable" - if counts["comment_rows"]: - summary += f", {counts['comment_rows']} comment rows captured" - elif comment_metrics and counts["posts"]: - summary += ", no comment engagement was readable" - if counts["partial"]: - summary += f", {counts['partial']} profile-only (posts not readable)" - if counts["blocked"]: - summary += f", {counts['blocked']} blocked" - note = cap_note(capped, missing) - if note: - summary += f". {note}" - state = "partial" if state != "error" else state - steps["write"] = "partial" - elif not dry: - steps["write"] = "ok" - if master_note: - summary += f". {master_note}" - state = "partial" if state != "error" else state - steps["write"] = "partial" - if dry: - summary = f"Test run. Nothing saved. Would have written: {summary}" - steps["write"] = "skipped" - state = "partial" if state != "error" else state - if notes: - summary += f". {notes[0][:120]}" - if not dry: - # C7: the fresh master rows are exactly what the table's metric cells summarise. - try: - compute_metric_cells(rt, table_key) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - return (state, summary, counts, affected, steps) - - -def _auto_url_field(table, fkey): - """The URL field an automation column is bound to: its own `automation.urlField`, else the - first url-typed column on the table. Never guessed silently — the caller reports which.""" - for f in table.get("fields") or []: - if f.get("key") == fkey: - bound = ((f.get("automation") or {}).get("urlField") or "").strip() - if bound: - return bound - for f in table.get("fields") or []: - if f.get("type") == "url": - return f.get("key") - return "" - - -#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge -#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a -#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own -#: cheaper approximation. -TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", - "first_found", "last_found", "source", "source_payload"}) - - -def _tt_candidate_row(row, stamp): - """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. - - ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so - writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as - `_candidate_row` on the Instagram side. - ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what - the enrich path will use too, so a corpus row and a scrape row cannot disagree about which - vendor key becomes which column. - """ - import connectors_tt as _tt - cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) - if not str(cells.get("handle") or "").strip(): - return None - return {**cells, "last_found": stamp, "source": "corpus"} - - - -def _ig_discovery_series(rt, table_key, incoming, stamp, username, auto_id, counts, log): - """⭐⭐ THE FORWARD HALF (2026-08-10), Instagram only. Every candidate a run wrote also leaves - an OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the next - discovery run re-opens exactly the gap the backfill just closed, which is how a data defect - becomes a recurring one. - - ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change worth - owning: a tenant that has never enriched gets the four canonical child databases the first time - discovery runs (four of `MAX_UT_TABLES = 40`). The alternative writes a series into a store the - profile table has no LINK to — the numbers would be recorded and unreachable, which is half a - feature wearing a whole one's clothes. - ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not to every - row the table happens to hold. - """ - try: - graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) - counts["series"] = append_ig_snapshots( - rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], - graph[IG_SNAPSHOTS_TABLE], log=log) - except Exception as e: # noqa: BLE001 - # The candidates already landed. A series write that fails must not throw away the rows we - # just paid the vendor for — the same posture the preset top-up takes. - log(f"[aios-auto] discover {auto_id}: corpus series write failed: " - f"{type(e).__name__}: {e}") - - -def _ig_discovery_metrics(rt, table_key, auto_id, log): - """C7 — the metric FIELDS computed off the master series. See the field runner.""" - try: - compute_metric_cells(rt, table_key) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - - -#: ⭐⭐ WAVE 30 · T09 (DEBT D-129) — ONE DISCOVERY RUNNER, TWO CORPORA, AND EVERY DIFFERENCE BETWEEN -#: THEM IS A ROW IN THIS TABLE. -#: -#: `run_discover_tiktok` was written in wave 29 as a deliberate SIBLING of `run_discover_instagram` -#: — its own docstring said so and gave the reason: *"it would rewrite the one code path this -#: product bills money through"*. That was the right call with one platform's worth of evidence and -#: it stopped being right when the duplication reached ~210 lines, because a sibling does not -#: inherit fixes. The proof is already in the copy below: the TikTok runner carries the **D-116** -#: precedence rule (a corpus re-find must not overwrite an ENRICHED measurement with the corpus's -#: cheaper approximation) and the Instagram one, which is the one with paying rows in it, does not. -#: -#: ⛔ **THE COLLAPSE IS BEHAVIOUR-PRESERVING, DELIBERATELY, AND `discovery_keys` IS WHERE YOU CAN -#: SEE THAT DECISION RATHER THAN INFER IT.** Setting Instagram's to `IG_DISCOVERY_KEYS` would close -#: D-116 in one line — and it would change what a live, billed, nightly automation writes to rows -#: the owner reads, inside a refactor whose whole claim is that nothing moved. It is REPORTED -#: instead (see D-116's row), which is the standing rule for a limit that is not being removed -#: today. ⚠ The one-line fix is now visible in a table instead of buried 300 lines apart, which is -#: most of the value of doing this at all. -#: -#: ⚠ `dataset` IS A CALLABLE, not a string: `connectors_tt` imports this module at import time, so -#: the engine may only reach it from INSIDE a function. `mapper` has the same shape for the same -#: reason on the TikTok side. -#: ⚠ `handle_field` is the column the engine-added `not_in` exclusion names, and it is per platform -#: because B-20 MEASURED that Bright Data's TikTok Profiles dataset calls Instagram's `account` -#: `account_id`. The exclusion never appears as a condition row, so nothing a person types can -#: correct it. -DISCOVERY_SPECS = { - "discover_instagram": { - "noun": "profile", - "log_tag": "discover", - "platform": PLATFORM_INSTAGRAM, - "dataset": lambda: BD_DS_PROFILES, - "handle_field": "account", - "mapper": lambda row, stamp: _candidate_row(row, stamp), - "table": DISCOVER_TABLE, - "label": DISCOVER_LABEL, - "fields": CANDIDATE_FIELDS, - # ⛔ D-116 IS OPEN ON THIS SIDE. `None` = discovery overwrites whatever it re-finds. - "discovery_keys": None, - "on_write": _ig_discovery_series, - "after_write": _ig_discovery_metrics, - }, - "discover_tiktok": { - "noun": "TikTok profile", - "log_tag": "discover-tt", - "platform": PLATFORM_TIKTOK, - "dataset": lambda: _tt_module().TT_DS_PROFILES, - "handle_field": "account_id", - "mapper": lambda row, stamp: _tt_candidate_row(row, stamp), - "table": TT_PROFILE_TABLE, - "label": TT_TABLE_LABELS[TT_PROFILE_TABLE], - "fields": TT_PROFILE_FIELDS, - "discovery_keys": TT_DISCOVERY_KEYS, - # ⚠ NO series and NO metric fields on this side, and that is not an oversight: both hooks - # write into Instagram's own snapshot store, through Instagram's own vocabulary. TikTok's - # series is W29-T06's `ut_tt_post_snapshots`, which the ENRICH path owns. - "on_write": None, - "after_write": None, - }, -} - - -def _tt_module(): - """`connectors_tt`, imported LAZILY. It imports this module at module level, so a top-level - import here would close the cycle.""" - import connectors_tt as _tt - return _tt - - -def run_discovery(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """Automation #3 (owner ruling R7 / D-23) and DEBT D-9: FIND handles nobody here has typed in, - on whichever corpus this automation's KIND names. - - Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus - query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every - outcome is a sentence about what actually happened; none of them is a green zero. - - ⛔ THE PLATFORM COMES FROM `defn["kind"]`, WHICH IS THE ONLY THING THAT DECIDES IT NOW. Before - wave 30 the choice was made by WHICH FUNCTION `RUNNERS` pointed at, so a fixture (or a stored - definition) with a stale kind still ran the corpus its caller had in mind. It cannot any more: - an unknown kind is refused rather than defaulted, because defaulting here means billing one - platform's corpus for another platform's search. - """ - kind = str(defn.get("kind") or "") - spec = DISCOVERY_SPECS.get(kind) - cfg = defn.get("config") or {} - dry = bool(cfg.get("dryRun")) - limit = int(cfg.get("recordsLimit") or 5) - preds = list(cfg.get("predicates") or []) - auto_id = str(defn.get("id") or "") - steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} - counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, - "missing_tables": 0} - if spec is None: - # ⛔ NEVER A DEFAULT. `RUNNERS` maps exactly the kinds in `DISCOVERY_SPECS` onto this - # function, so reaching here means somebody widened one of the two without the other. - steps["find"] = "blocked" - return ("error", f"this automation's kind ({kind or 'blank'}) has no profile corpus, so " - f"nothing was searched and nothing was charged", counts, [], steps) - noun = spec["noun"] - - if not bd_ready(): - # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is - # the vendor's — so this is where the run stops, saying which env var is missing. - steps["find"] = "blocked" - return ("partial", "Profile search is not set up yet. Nothing was searched and nothing " - "was charged", counts, [], steps) - - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - if not pending: - # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and - # the vendor bills for breadth whether the filter was saved yesterday or last month. - # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config - # stored before this check existed must not reach the vendor either. - # ⭐ W32-T46: the RUN-time twin of the save-time guard says the right network's name too — - # `defn["kind"]` is what decides the corpus everywhere else in this function. - guard_err = (narrowing_refusal(preds, cfg.get("operator"), str(defn.get("kind") or "")) - or depth_refusal(preds, cfg.get("operator"))) - if guard_err: - steps["find"] = "blocked" - return ("error", f"the search was not started. {guard_err}", counts, [], steps) - if dry: - est = discover_estimate(limit) - steps["find"] = steps["write"] = "skipped" - return ("partial", - f"Test run. Nothing saved. Would look for up to {limit} " - f"{noun}{'' if limit == 1 else 's'} matching " - f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " - f"${est['usd']}", counts, [], steps) - step("Starting the search") - # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never - # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the - # vendor refuses. Reported either way, because "why did this run find fewer" and "why did - # this run cost the same as last night" are both questions this line answers. - # ⚠ `already_found_handles` NEEDS NO PER-PLATFORM BRANCH — it is scoped to THIS definition's - # own tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` - # excludes exactly the handles it has itself found. Verified rather than assumed: the - # function reads `config.targetTable` plus each `create_record` action's table, and names - # no IG constant. - excl = {} - sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, - dataset_id=spec["dataset"](), - exclude_handles=already_found_handles(rt, defn), - applied=excl, handle_field=spec["handle_field"]) - if excl.get("excluded"): - step(f"excluded {excl['excluded']} already-found " - f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") - elif excl.get("dropped"): - step(f"searching without the already-found list. {excl['dropped']}") - if note: - steps["find"] = "blocked" - return ("error", f"the search was not started. {note}", counts, [], steps) - pending = sid - # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes - # through `rt.update`, which can RAISE on an unavailable store — and that exception - # becomes an error run, so a `log()` placed after it never executes. The snapshot would - # then exist, be billed, and have its id in neither the store nor the logs. Logging first - # means the worst case is still recoverable by a human reading the Space output. - log(f"[aios-auto] {spec['log_tag']} {auto_id}: started {sid}") - # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop - # existing because this process dies thirty seconds later, and a lost id is a set we - # paid attention to and can never collect. - set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) - steps["find"] = "ok" - - waited, status, size, note = 0.0, "", 0, "" - while True: - # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to - # BD_FILTER_WAIT at the vendor, and D measured that this wait IS the whole of "Run once now - # is laggy / looks stuck". A counter here is what makes a legitimate wait distinguishable - # from a hung thread; without it the rendered step reads the same word for both, which is a - # progress indicator that cannot indicate progress. - step(f"Searching. {int(waited)}s of " - f"{int(BD_FILTER_WAIT)}s") - status, size, note = bd_filter_status(pending) - if note: - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", note, counts, [], steps) - if status != "building" or waited >= BD_FILTER_WAIT: - break - time.sleep(BD_FILTER_POLL) - waited += BD_FILTER_POLL - step("Reading the profiles the search found") - - if status == "building": - # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 - # minutes; holding a worker thread that long on a free-tier container to poll would be - # the wrong shape. Say plainly that it is still running and who collects it. - # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made - # this read as "stuck": *"The next run picks up the results"* is a promise about a run - # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick - # finish it, so the promise is now kept by something rather than by the reader. - # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by - # naming the corpus size. It was TRUE, and it was answering a question the person had not - # asked. It was also the one place a vendor's catalogue size was quoted to a customer, so - # it would have gone stale the day the vendor grew. THE SHAPE OF THE ANSWER SURVIVES and is - # the part that mattered: the wait does not shrink when you ask for fewer records, and a - # person who does not know that reads "20 minutes for 10 rows" as a fault. - mins = 0 - try: - since = (defn.get("state") or {}).get("pendingSince") - if since: - mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) - - _dt.datetime.fromisoformat(str(since))).total_seconds() - // 60)) - except Exception: # noqa: BLE001 - mins = 0 - been = f". {mins} min so far" if mins else "" - steps["collect"] = "partial" - return ("partial", - f"Still searching at the provider{been}. A corpus search takes about 20 minutes " - f"however few records you asked for. Nothing is lost, you are not charged twice, " - f"and the results are collected automatically as soon as they are ready", - counts, [], steps) - if status == "empty": - # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword - # that is too specific, so it says what to do about it instead of reporting a fault. - steps["collect"] = "ok" - steps["write"] = "skipped" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("partial", "No profiles matched. Try a shorter or more common keyword. " - "'floral' finds more accounts than 'floral design studio'", - counts, [], steps) - if status != "ready": - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", - counts, [], steps) - - rows, dnote = bd_filter_rows(pending) - if dnote: - # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. - steps["collect"] = "partial" - return ("partial", f"the search finished with {size} match" - f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - steps["collect"] = "ok" - - stamp = _iso() - _map = spec["mapper"] - incoming = [c for c in (_map(r, stamp) for r in rows) if c] - counts["found"] = len(incoming) - # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and - # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to - # "the vendor delivered 3". It also catches the shape where an error document comes back - # down the rows path and parses as one unusable row instead of an error. - counts["dropped"] = max(0, len(rows) - len(incoming)) - label = cfg.get("targetLabel") or spec["label"] - table_key = (ut_key_for(label, cfg.get("targetTable") or spec["table"]) if dry - else ut_ensure(rt, label, spec["fields"], username, - key=cfg.get("targetTable") or spec["table"], - flow_tag=auto_id, lock_fields=True)) - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`. - # - # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning - # that built it is worth stating before it is discarded rather than after: per-user rows gave - # each person their own `found_count` and their own review card, so two colleagues scouting the - # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The - # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no - # longer part of the identity. - # - # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN - # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on - # TikTok are routinely different people, so keying on the handle ALONE would have silently - # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with - # no error, discovered months later by someone wondering why a creator's followers halved. - # Hence `platform`, and hence it being part of the key rather than a label beside it. - _ck = candidate_key - - # ⚠ A ROW WITH NO `platform` CELL IS STAMPED WITH **THIS CORPUS'S** PLATFORM, and the default is - # per spec rather than global. On the Instagram table every legacy row came from the Instagram - # runner, so `Instagram` is a FACT about the stored data; in a `ut_tt_*` table the same blank - # means TikTok for exactly the same reason. Blank-keyed rows would fail to match their own - # re-find and duplicate the whole table on the next run. - for r in existing.values(): - if not str(r.get("platform") or "").strip(): - r["platform"] = spec["platform"] - existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))} - for rid, r in existing.items()} - # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding - # a profile is the signal that it keeps matching, so the counter grows instead of - # resetting. `first_found` is written only when the pair is new, so it never moves. - seen = {r["_ckey"]: r for r in existing2.values()} - _dkeys = spec["discovery_keys"] - for c in incoming: - # "Found by" — the first finder is recorded and later finders do not overwrite them, which - # is what the column means now that it is no longer part of the identity. - c["created_by"] = username - c["_ckey"] = _ck(c.get("platform"), c["handle"]) - prev_row = seen.get(c["_ckey"]) - if prev_row and str(prev_row.get("created_by") or "").strip(): - c["created_by"] = prev_row["created_by"] - if prev_row: - counts["seen_again"] += 1 - c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) - # ⛔⛔ DEBT D-116 — THE ENRICHED-MEASUREMENT PRECEDENCE RULE, AND IT RUNS ONLY WHERE THE - # SPEC DECLARES A `discovery_keys` SET. A discovery re-find OVERWRITES an enriched - # `followers` with the corpus number — the corpus is cheaper, rounder and older than a - # scrape, so the row silently gets WORSE every night while the automation reports - # success. The fix is a precedence rule, not a blank check: a row that has been ENRICHED - # (`enriched_at` is stamped only by the enrich path) keeps its exact measurements, and - # discovery may only fill what is genuinely empty and update its own bookkeeping. - # ⚠ The declared set is what discovery OWNS; everything else on a corpus row is a - # measurement, and an approximation must never replace an exact one. - if _dkeys and str(prev_row.get("enriched_at") or "").strip(): - for k in list(c): - if (k not in _dkeys and k != "_ckey" - and str(prev_row.get(k) or "").strip()): - c.pop(k) - else: - counts["new"] += 1 - c["found_count"] = "1" - c["first_found"] = stamp - # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF - # "is this row new to the TABLE". - # - # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's - # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true, - # and then drew nothing, because this stamp lived inside the `else` above. `skey` is - # `stage_`; the branch it sat in asks whether some OTHER automation had - # already created the row. So the first automation to reach a handle claimed it, and every - # later automation sharing that database silently had no cards — with a summary saying it - # had found them. - # - # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an - # unconditional write: a re-find must never pull a card somebody has already moved back - # into Review. So the question is "has THIS automation placed this record yet?" — not "is - # this record new?" A blank stage cell for this automation means unplaced, which is - # exactly the state a new card is in. - merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) - merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} - for rid, r in merged.items()} - counts["capped"] = mc["capped"] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - if not dry and not missing: - ut_write_rows(rt, table_key, merged) - steps["write"] = "ok" - if spec["on_write"]: - spec["on_write"](rt, table_key, incoming, stamp, username, auto_id, counts, log) - elif dry: - steps["write"] = "skipped" - - affected = [rid for rid, r in merged.items() - if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] - est = discover_estimate(counts["found"]) - summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found. " - f"{counts['new']} new, {counts['seen_again']} seen before. About " - f"${est['usd']}") - if counts["dropped"]: - summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " - f"username and could not be saved") - cnote = cap_note([(table_key, counts["capped"])], missing) - if cnote: - summary += f". {cnote}" - steps["write"] = "partial" - if dry: - summary = f"Test run. Nothing saved. Would have written: {summary}" - state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" - if not dry and not missing and spec["after_write"]: - spec["after_write"](rt, table_key, auto_id, log) - log(f"[aios-auto] {spec['log_tag']} {auto_id} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -def _predicate_sentence(preds, operator="and"): - """A predicate list as something a person can read back. Used in summaries and the canvas. - - Human words on BOTH halves — it used to render `biography includes floral`, the vendor's - column name beside the vendor's operator token, in a sentence shown to a customer. - """ - joiner = " or " if str(operator or "").lower() == "or" else " and " - parts = [] - for p in preds or []: - v = p.get("value") - shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v - parts.append(f"{field_label(p.get('name'))} " - f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}" - + ("" if shown is None else f" {shown}")) - return joiner.join(parts) or "no conditions" - - -#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather -#: than a second, smaller number nobody could explain: a plain automation's records ARE its -#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a -#: flow could be pointed at and which would not finish. -#: ⚠ W31-T36 (D-143 / R6) — THIS IS NOW THE FALLBACK ONLY: what a flow may walk when the table's -#: own limit cannot be resolved (no runtime, or a key `core.user_tables` does not know). The real -#: answer is per TABLE and comes from `core.user_tables.row_limit`; see `_flow_record_cap`. -#: ⛔ Do not re-point a caller at this constant — a private copy of "is this database bounded" is -#: exactly the second evaluator D-143 was. -MAX_FLOW_RECORDS = MAX_UT_ROWS - - -def _flow_record_cap(table_key, rt=None): - """How many records may a flow walk in this database? `None` means NO CAP (R6). - - ⛔ DELEGATES, NEVER DECIDES. `core.user_tables.row_limit` is the one evaluator for "is this - database bounded", and it distinguishes three states this module must not re-derive: `0` - (read-through — its rows are not in the document at all), `None` (connected and UNCAPPED, which - is R6's whole point), and `MAX_ROWS` (the editable substrate, genuinely bounded by the shared - document). A second copy here is exactly the defect D-143 IS. - - ⚠ `0` MEANS "NO ROWS LIVE HERE", NOT "WALK NOTHING". A read-through grid's rows arrive through - the mirror route, so a flow that reached this function already holds ids from somewhere else; - capping it at zero would refuse a walk whose records are in hand. Treated as uncapped, and the - honest place to fix a read-through walk is the walker. - """ - try: - import core.user_tables as _ut_cap - cap = _ut_cap.row_limit(str(table_key or ""), st=rt) - except Exception: # noqa: BLE001 - return MAX_FLOW_RECORDS # the fallback, and it is the conservative direction - return None if not cap else int(cap) - - -def _flow_cap_reason(table_key, rt=None): - """R6's SECOND sentence, appended to the run's own note: the cause and the recommended fix. - - ⛔ R6 IS TWO SENTENCES AND THE SECOND IS THE ONE THAT GETS DROPPED — *"if there is lag or it - can't be done, you need to explicitly tell me why and recommend a fix."* A run that says only - *"walked the first 5000"* has disclosed the number and hidden everything a person could act on. - `core.user_tables.limit_report` already carries both, so this READS them rather than writing a - second wording that would drift from the one the grid shows. - """ - try: - import core.user_tables as _ut_cap - rep = _ut_cap.limit_report(str(table_key or ""), st=rt) or {} - except Exception: # noqa: BLE001 - return "" - cause, fix = str(rep.get("cause") or ""), str(rep.get("recommendation") or "") - if not cause and not fix: - return "" - return f". {cause}." + (f" To walk them all: {fix}." if fix else "") - - -# --------------------------------------------------------------------------------------------- -# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE STATEMENTS BATCH: ASSEMBLE, PARK, NEVER SEND -# -# ⛔⛔ THE TICKET'S `how:` SAID TO USE "the board's strict review posture (wave 22)". THAT POSTURE -# DOES NOT EXIST, and rebuilding it would REVERSE AN OWNER RULING rather than merely miss a symbol. -# Wave 27 R3 deleted `board()`, `move_card()`, `stages_for()`, `ensure_stage_field()` and the review -# branch of the action walk (the tombstone is above `LANE_OPS`), and its reason is a DATA reason, -# verbatim: *"the board wrote MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` -# stamp and a `_cycles` counter per automation — to render a view of state the RUN LOG already -# holds."* `retire_automation_stage_fields` and `migrate_ig_tables` still DROP those columns, so a -# batch parked in a stage field would be deleted by our own migration. -# ⇒ The batch parks on the AUTOMATION DEFINITION, the shape `runs` and `reviews` already use. Zero -# machine columns on a customer's database, and the migration has nothing to take away. -# -# ⛔ A STATEMENTS FLOW IS **BATCH**-SCOPED, NOT RECORD-SCOPED, and that is why it does not go -# through `apply_actions` at all. `run_plain` walks every row of the flow's bound table, so a -# per-record arm would assemble the whole worklist once per record; and a statements agent binds NO -# table (its customers come from the Odoo AR worklist, not a `ut_*` grid), so `run_plain` would -# have answered `partial: "no database is bound yet"` and the step would NEVER HAVE RUN. The -# precedent for a kind that defines its own scope is already here: the enrich-only branch below, -# whose saved View "ARE the set of records the automation was asked to process". -# ⚠ The `_walk` arm for this kind therefore stays a REFUSAL, not a duplicate: a `send_statement` -# dropped into an ordinary record-walking flow must say it did nothing, never half-send. -# -# ⛔⛔ AND NOTHING HERE SENDS. Not one function in this section imports the mail path. The send door -# is `routes_statements`, behind `admin_gate` + `_royal_only`, calling `collections_send. -# queue_statement`, whose SAFE_MODE guardrail lives in the DATA LAYER where no route, payload or UI -# can bypass it. This module ASSEMBLES and PARKS. A person clicks Send. - -#: How many statements one parked batch holds. Bounded for the reason `MAX_RUNS` and `MAX_REVIEWS` -#: are: this rides inside the automation DEFINITION, and an unbounded list is a serialisation cost -#: on every read of the automations bucket. -#: ⚠ WHEN IT BITES IT IS DISCLOSED, never a silent `[:N]` — R6's second sentence, and -#: [[no-unverifiable-aggregates]]. `assemble_statements` appends a note naming the number left out. -MAX_STATEMENT_BATCH = 200 - -#: The collection worklist loader, injectable so a gate can drive this without Odoo credentials. -#: Production leaves it None and the real data layer answers. Same shape as `_DRAFT_CHAT`. -_STATEMENT_ROWS = [None] - - -def _collections(): - """`modules.collections_send`, imported lazily — `platform/` is not on the path at import time - for every consumer of this module, and this is the only section that needs it.""" - import modules.collections_send as cs # noqa: PLC0415 - return cs - - -def statement_steps(defn): - """The ENABLED `send_statement` actions of this flow, top level only. - - ⚠ Top level only, deliberately, and `is_statement_flow` is what makes that safe: a batch flow - is exactly one enabled action. A `send_statement` nested inside an If is NOT a batch flow, so - it falls to the ordinary record walk and is refused there with a sentence. - """ - return [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) - if isinstance(a, dict) and a.get("kind") == "send_statement" - and a.get("enabled", True)] - - -def is_statement_flow(defn): - """Is this automation a statements BATCH rather than a record walk? - - ⛔ EXACTLY ONE ENABLED ACTION, AND IT IS THIS KIND. The narrowness is the safety: a flow that - also updates records has record semantics that the batch path would silently drop, so it keeps - the ordinary walk (where the arm refuses and says so). This mirrors the enrich-only branch in - `run_plain`, which is narrow for the same stated reason. - """ - enabled = [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) - if isinstance(a, dict) and a.get("enabled", True)] - return len(enabled) == 1 and enabled[0].get("kind") == "send_statement" - - -def assemble_statements(defn, log=print): - """`(batch, notes)` — the statements this configuration WOULD send, rendered, plus why anybody - was left out. **Reads Odoo read-only. Sends nothing. Writes nothing.** - - Each entry is `{customer, to, subject, html, tier, overdue}`. `html` is rendered by the SAME - `render_statement_html` the send door uses, so what a person approves is what goes out — a - review screen rendering its own approximation of the mail is a review of the wrong thing. - - ⚠ A CUSTOMER WITH NO EMAIL IS *SKIPPED AND NAMED*, never dropped. `routes_statements.send` - already separates "we could not" from "there was nowhere to send"; this keeps that distinction - at the assembly end, because a batch that silently shrinks is one nobody can reconcile. - """ - cfg = (statement_steps(defn) or [{}])[0].get("config") or {} - cs = _collections() - notes = [] - loader = _STATEMENT_ROWS[0] - rows = list(loader() if loader else cs.load_collection_list(cs.Odoo())) - tier = str(cfg.get("tier") or "").strip() - if tier: - before = len(rows) - rows = [r for r in rows if str(r.get("Tier") or "") == tier] - log(f"[aios-auto] statements: {len(rows)} of {before} customers are in tier {tier}") - subject_tpl = str(cfg.get("subject") or "") or cs.DEFAULT_SUBJECT - intro_tpl = str(cfg.get("intro") or "") or cs.DEFAULT_INTRO - footer_tpl = str(cfg.get("footer") or "") or cs.DEFAULT_FOOTER - month = _dt.date.today().strftime("%B %Y") - batch, no_email = [], [] - for row in rows: - if len(batch) >= MAX_STATEMENT_BATCH: - break - name = str(row.get("Customer") or "") - to = str(row.get("Email") or "").strip() - if not to: - no_email.append(name) - continue - try: - subject = subject_tpl.format(customer=name, company=cs.COMPANY, month=month) - except (KeyError, IndexError): - # An unknown placeholder is the person's typo, not a crash. Show what they typed — - # the same choice `routes_statements.preview` already makes. - subject = subject_tpl - batch.append({"customer": name, "to": to, "subject": _s(subject, 200), - "html": _s(cs.render_statement_html(row, intro_tpl, footer_tpl), 40000), - "tier": str(row.get("Tier") or ""), "overdue": row.get("Overdue")}) - if no_email: - shown = ", ".join(no_email[:5]) - one = len(no_email) == 1 - notes.append(f"{len(no_email)} customer{'' if one else 's'} " - f"{'has' if one else 'have'} no email address on their record and " - f"{'is' if one else 'are'} not in this batch " - f"({shown}{', and others' if len(no_email) > 5 else ''}). " - f"Add an address in Odoo and run it again") - left = len(rows) - len(batch) - len(no_email) - if left > 0: - # R6's second sentence: a limit that bites is REPORTED, with its cause and what to do. - notes.append(f"{left} more customers matched but one batch holds " - f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest, or " - f"narrow the tier filter") - return batch, notes - - -def park_statements(rt, auto_id, batch, notes): - """Write the assembled batch onto the DEFINITION as `pendingStatements`. Replaces, never - appends: a batch is THIS run's worklist, and two runs' statements merged into one list is a - customer invoiced twice. - - ⚠ `flush="async"` for the reason every writer in this module gives — the read-your-writes - contract means the next `GET /automations` already sees it, and only the upload is deferred. - """ - entry = {"ts": _iso(), "count": len(batch), "sent": False, - "items": list(batch)[:MAX_STATEMENT_BATCH], - "notes": [_s(n, 300) for n in (notes or [])][:25]} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d["pendingStatements"] = entry - return cur - - _store_update(rt, _up, flush="async") - return entry - - -def clear_statements(rt, auto_id): - """Drop a parked batch — after it is sent, or when somebody discards it.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d.pop("pendingStatements", None) - return cur - - _store_update(rt, _up, flush="async") - - -def run_statements(rt, defn, username="automation", log=print, step=_no_step): - """The batch runner: assemble, park, tell somebody. Returns `run_plain`'s 5-tuple. - - ⛔ THE STATE IS `partial`, NEVER `ok`, AND THAT IS DELIBERATE. `ok` would put a green dot over - a job that is only half done — the statements exist and NOBODY HAS SENT THEM. `partial` is this - module's honest-progress state and the summary says exactly what is waiting, which is what makes - the run notification (`_notify_run`) read as "come and look" rather than "done". - """ - if not is_statement_tenant(rt): - # Belt and braces behind `clean_actions`' wall: a definition stored before the wall, or - # copied between tenants, must not assemble another company's customer list. - return ("error", "statements are not configured for this workspace", {}, [], {}) - step("reading the collection list") - try: - batch, notes = assemble_statements(defn, log=log) - except Exception as exc: # noqa: BLE001 - return ("error", f"the collection list could not be read: {str(exc)[:200]}", {}, [], {}) - step("rendering statements") - park_statements(rt, defn.get("id"), batch, notes) - counts = {"statements": len(batch)} - if not batch: - return ("partial", "no customers matched, so there is nothing to review", counts, [], {}) - one = len(batch) == 1 - return ("partial", - f"{len(batch)} statement{'' if one else 's'} " - f"{'is' if one else 'are'} ready for review. Nothing has been sent. Open this agent " - f"and click Send to release them", - counts, [], {}) - - -#: ⭐⭐ WAVE 35 · T37 — THE SEEDED ROYAL IMPORTS STATEMENTS AGENT. -#: A FIXED id, because the row's own EXISTENCE is the idempotency key (see `seed_statements_agent`). -STATEMENTS_AGENT_ID = "system:statements" -SYSTEM_STATEMENTS = "statements" -#: First of the month, 06:00. A monthly statement run is what the collections worklist is for, and -#: the hour matches the Odoo sync default so the numbers it reads are the night's. -STATEMENTS_DEFAULT_CRON = "0 6 1 * *" - - -def seed_statements_agent(rt, username="system"): - """Mint Royal Imports' "Monthly statements" agent ONCE, **switched OFF**. Returns the - definition if it wrote one, else None. - - ⛔⛔ IT ARRIVES `enabled: False` AND THAT IS THE TICKET'S OWN TRAP, not a preference. An agent - that ships enabled has scheduled itself against real customers before a single person has read - its configuration — and this one's action queues mail to the tenant's actual debtors. - - ⭐ WHY STORED, WHEN WAVE 34's SYSTEM AGENT IS DERIVED. `_odoo_sync_row` gives three reasons for - deriving and only one survives contact with THIS agent (raised as `ASK D-11`): - · "two copies of one cadence" — does not apply: Odoo's cadence already lives in the connector - config, so a stored copy could drift; a statements schedule has no other home to drift from. - · "stored means the failure-pause can disable it" — INVERTS: auto-pausing statements after K - consecutive failures is correct (a dead credential should stop it), where pausing the Odoo - cadence would silently break infrastructure. - · "nothing can seed it" — real, and answered here: ONE write per tenant ever, by the - CONTAINER (which is what D-195 asks for; what it forbids is a CLI writing while the Space - is up), behind the existence check below. - And deriving cannot meet T37 anyway: a derived row is not in the automations bucket, so edit, - toggle, schedule and run would each need a bespoke door — four new mechanisms to avoid one - guarded write. The Odoo agent escapes that only because a separate resync loop already does its - work. - - ⛔ THE EXISTENCE CHECK IS THE WHOLE IDEMPOTENCY STORY, AND IT IS SAFE ONLY BECAUSE THE ROW - CANNOT BE DELETED. `routes_automation.delete_automation` refuses any stored row carrying - `system` (409, with a sentence), so "the row is present" can never become false behind our - back. If that refusal is ever relaxed, THIS FUNCTION NEEDS A SEPARATE DURABLE FLAG — otherwise - deleting the agent resurrects it on the next list call, which is a delete that undoes itself. - ⚠ And `system` is STICKY through `clean_definition` (see its return) for the same reason: an - edit that stripped the marker would make the row deletable and re-open exactly that hole. - """ - if not is_statement_tenant(rt): - return None - existing = all_definitions(rt) or {} - if STATEMENTS_AGENT_ID in existing: - return None - raw = { - "name": "Monthly statements", - "kind": "plain", - # ⚠ A SCHEDULE THAT IS OFF, not an absent schedule: the cron is the CONFIGURATION T37 asks - # a person to open and read, and a blank one would make them invent it before they could - # judge it. - "schedule": {"cron": STATEMENTS_DEFAULT_CRON, "enabled": False}, - "trigger": {"key": "schedule"}, - "flow": {"actions": [{ - "id": "act_1", "kind": "send_statement", - # ⚠ TIER BLANK = every tier, which is the honest default: narrowing to "A-Urgent" would - # be us deciding who gets chased, and the blank is the value the config panel shows as - # "all of them" rather than an empty box. - "config": {"tier": "", "subject": "", "intro": "", "footer": ""}, - }]}, - } - defn, err = clean_definition(raw, None, username, rt=rt) - if err: - return None - defn["id"] = STATEMENTS_AGENT_ID - # Stamped AFTER the clean, because `clean_definition` reads this key from `prev` only — a - # payload may never assert it (see that function's note). - defn["system"] = SYSTEM_STATEMENTS - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - # ⚠ RE-CHECKED INSIDE THE MUTATION. Two concurrent list calls can both pass the read above; - # the store applies mutations under its own lock, so this is where "only once" is actually - # decided. Without it the second writer would overwrite a row a person may already have - # edited, silently resetting their schedule and template. - if STATEMENTS_AGENT_ID not in cur: - cur[STATEMENTS_AGENT_ID] = defn - return cur - - _store_update(rt, _up, flush="async") - return defn - - -def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole - automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special - case and `apply_actions` still runs at the ONE call site that already exists. - - Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed - `error: unknown automation kind 'plain'` — on the kind every new automation now has. - - ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here. - `apply_actions` returns immediately when there is no table, so a flow whose only action is - `create_record` into some other database does NOTHING — and a green dot over nothing is the - exact defect this module names in three other places. It reports the honest state and says - which fact is missing. - """ - # ⭐⭐ WAVE 35 · T36 — THE BATCH FLOW BRANCHES BEFORE THE TABLE CHECK, and the ORDER is the - # whole of it. A statements agent binds no `ut_*` table (its customers come from the Odoo AR - # worklist), so leaving this below the `if not table` return would answer "no database is bound - # yet" and the step would never run — a feature that is whole, gated and unreachable, which is - # this repo's most-repeated failure. See the section above `run_plain` for why it is not an - # `apply_actions` arm. - if is_statement_flow(defn): - return run_statements(rt, defn, username=username, log=log, step=step) - table = _flow_table(defn) - if not table: - return ("partial", - "no database is bound yet. A plain automation walks the records of its target " - "database, and this one names none (pick one on the trigger, or in Properties)", - {}, [], {}) - all_rows = (ut_get(rt, table) or {}).get("rows") or {} - # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug - # on the kind every new automation now has. `plain` has no machine step, so it has nothing of - # its own to call "the records this run touched" — and walking the WHOLE table would mean - # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the - # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them - # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what - # those genuinely mean. - ids = ([r for r in (rows or []) if str(r) in all_rows] if rows - else sorted(all_rows, key=_rid_num)) - - # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its - # saved View and quota are not merely how the action decides whether to spend; they ARE the - # set of records the automation was asked to process. Walking every row first meant a run - # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows. - # Besides being misleading, that shape made the runtime scale with unrelated historical rows. - # - # Keep this deliberately narrow: any flow with another action must still hand that action the - # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record - # semantics to preserve, so it can begin at its selected records safely. - flow_actions = list((defn.get("flow") or {}).get("actions") or []) - enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)] - enrich_scope_note = "" - if rows is None and len(enabled_direct) == 1 \ - and enabled_direct[0].get("kind") == "enrich_instagram": - enrich_cfg = enabled_direct[0].get("config") or {} - profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField")) - if profile_key: - # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES. - # `enrich_selection` is called TWICE per run — here, to decide which records the - # runner walks and announces, and again inside `_walk` to decide which the action - # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so - # the run announced *"(1 of the 30 asked for)"* in its summary while its own note - # from the second call said *"0 of the 30"* — one run, two answers, both printed. - # ⚠ `gone` is read from the definition here rather than passed in, because this - # function has the definition and `_walk` does the same read; a third source of the - # same fact is how the two would drift again ([[one-evaluator-per-question]]). - selected, enrich_scope_note = enrich_selection( - rt, table, enrich_cfg, profile_key, - gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})) - # ⛔ D-112 — AND THE CONSTANT IS THE FIX, NOT THE `if`. This guard was a `startswith` - # against the sentence SPELLED OUT, twelve hundred lines from the only place that - # produces it. Reword the message in `enrich_selection` — a perfectly ordinary edit, - # since it is a sentence a person reads — and this line silently stops matching, the - # run goes back to reporting `ok`, and the defect D-112 describes returns with nothing - # anywhere going red [[gate-pins-a-spelling-not-a-claim]], on the product side. - # ONE constant, two readers, so the wording is free to change and the behaviour is not. - if enrich_scope_note.startswith(ENRICH_VIEW_UNREADABLE): - return ("partial", f"nothing walked in {table}. {enrich_scope_note}", {}, [], {}) - ids = selected - step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") - counts = {"records": len(ids)} - # ⭐⭐ WAVE 31 · T36 (D-143, owner ruling R6) — THE CAP IS THE TABLE'S, NOT THIS MODULE'S. - # - # ⛔ WHAT WAS WRONG, and it was a SECOND EVALUATOR rather than a wrong number. This line read - # `MAX_FLOW_RECORDS = MAX_UT_ROWS = 5000` — a constant mirrored from the EDITABLE substrate's - # ceiling — and applied it to every database alike. `ut_odoo_orders` is a CONNECTED source - # (32,826 rows live), and R6 is explicit that connected-source data has NO CAP: *"I thought we - # decided there is no cap in how many data from the API source … can be pulled into the app."* - # So a flow over it silently walked 5,000 of 32,826 — 15% of the records — and every total it - # produced understated the book while looking exactly like a complete run. - # - # ⭐ `core.user_tables.row_limit` IS THE ONE EVALUATOR for this question and it already answers - # the three cases (0 = read-through, None = connected and UNCAPPED, MAX_ROWS = editable). Using - # it here retires this module's private copy instead of correcting it — one question, one - # normaliser [[one-question-two-normalizers]]. `limit_report` is R6's SECOND sentence already - # expressed as data, so the sentence a person reads carries the cause and the recommended fix - # rather than just a number. - _cap = _flow_record_cap(table, rt) - if _cap is not None and len(ids) > _cap: - # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names - # both numbers so "it only processed some of them" is readable rather than deducible — and - # since T36, WHY it applies and what to do about it. - total = len(ids) - ids = ids[:_cap] - return ("partial", - f"{total} records in {table}. This run walked the first {_cap}" - + _flow_cap_reason(table, rt), counts, ids, {}) - n = len(ids) - # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK. - # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)? - # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours: - # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product - # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key. - # · `(all)` meant "every record, because a manual run has no trigger records to narrow to", - # which is unguessable from the word. It reads as a filter nobody chose. - # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole - # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that - # this sentence never claimed to report. What was wrong was that nothing said so. - label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table - scope = ("the records that fired the trigger" if rows - else ("the records selected for Instagram enrichment" - if len(enabled_direct) == 1 - and enabled_direct[0].get("kind") == "enrich_instagram" - else "every record. This run was started by hand, so nothing narrowed it")) - if enrich_scope_note: - scope += f" ({enrich_scope_note})" - # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400 - # characters by `_commit_run`, and this note is the longest thing a run says — it names - # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING - # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy - # of this sentence is never produced. Without this line the explanation exists only in a - # string that is about to be truncated, on exactly the runs that look like the automation - # has stopped working. - counts[RUN_NOTES_KEY] = [enrich_scope_note] - return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}", - counts, ids, {}) - - -RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, - "field_instagram": run_field_instagram, - # ⭐ WAVE 30 · T09 (D-129) — BOTH DISCOVERY KINDS MAP ONTO THE SAME FUNCTION. The kind is - # no longer chosen by which callable this dict holds; `run_discovery` reads it off the - # definition and looks the corpus up in `DISCOVERY_SPECS`. The two dicts are asserted - # key-for-key by the gate, because a kind in one and not the other is either an - # unroutable automation or a refused run. - "discover_instagram": run_discovery, - "discover_tiktok": run_discovery} - - -# --------------------------------------------------------------------------------------------- -# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) -# --------------------------------------------------------------------------------------------- -# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → -# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness -# boolean on the wire, four client strings and two gate sections. That is what a hard-coded -# vendor costs, and TikTok (D-9) would have paid it again from scratch. -# -# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the -# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: -# -# probe "is this configured?" -> the honest readiness bit the UI shows -# capture enrich a handle we know -> `pull_profile`-shaped -# discover find handles we do not -> the corpus query -# -# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in -# later; it is the honest statement that this source has no discovery route, and a surface that -# offers one anyway is offering a button that must refuse. -SOURCES = { - "instagram": { - # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing - # copy, and which provider serves a capability is a routing decision the product owns — - # `providers.py` still holds the real keys, chains and costs. - "key": "instagram", "label": "Instagram", "vendor": "Scraper", - "probe": bd_ready, - "capture": pull_profile, - "discover": bd_filter_start, - "captureKind": "field_instagram", - "discoverKind": "discover_instagram", - "tiers": TIERS, - }, - # ⭐⭐ WAVE 29 — D-9 LANDS, AND IT LANDED AS THE ONE ENTRY THIS REGISTRY PREDICTED IT WOULD. - # The canvas, the append tables, the caps and the honest-status contract were already in place; - # what TikTok added was a field map, a discovery runner and three dataset ids. - "tiktok": { - # `vendor` NAMES NO COMPANY (owner instruction 2026-08-09) — it reaches operator-facing - # copy, and which provider serves a capability is a routing decision `providers.py` owns. - "key": "tiktok", "label": "TikTok", "vendor": "Scraper", - "probe": bd_ready, - # ⭐ WAVE 30 · T08 — CAPTURE IS WIRED, and it is a LAZY reference rather than the function - # object every other row here holds. `connectors_tt` imports THIS module at module level, - # so naming `connectors_tt.pull_profile_tt` at import time is a circular import — which is - # why every existing site does `import connectors_tt` inside the function body. A - # zero-argument callable keeps the registry's shape (a reader gets a callable, not a - # string) without moving the import. - # ⚠ AND SETTING IT SWITCHES NOTHING ON. `SOURCES` is never subscripted anywhere; its one - # reader is `source_status()`, which reads `key/label/vendor/probe/discover`. An action - # kind is gated in three other places entirely (`ACTION_CATALOG`, `clean_actions`, - # `apply_actions`). This row is documentation that must not lie, not wiring. - "capture": lambda *a, **k: __import__("connectors_tt").pull_profile_tt(*a, **k), - "discover": bd_filter_start, - "captureKind": "enrich_tiktok", - "discoverKind": "discover_tiktok", - # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never - # had one. An empty tuple is the honest statement, not a gap to fill in later. - "tiers": (), - }, -} - - -def source_status(): - """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. - - ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface - needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the - credential to a browser would put a billable secret in every user's devtools. - """ - # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which - # supplier this rung actually uses — that history is worth keeping in the code — but a - # customer has no use for it, and a server-supplied string rendered by a generic component - # is precisely the half a client-file scan cannot see. - return [{"key": s["key"], "label": s["label"], - "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} - for s in SOURCES.values()] - - -# --------------------------------------------------------------------------------------------- -# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client -# --------------------------------------------------------------------------------------------- -# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are -# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — -# a canvas that still shows a step the engine stopped running looks completely fine. So the node -# list is derived from the DEFINITION by the same module that executes it, rides on the payload -# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. -# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. -# -# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is -# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine -# walks it linearly. Nothing here should be read as a general branching runtime. - -#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be -#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw -#: away the only information the dot is there to carry. -NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") - -#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a -#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is -#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones -#: here are all REAL — each changes what the next run does: -#: schedule the trigger fires on its cron, or only by hand -#: postMetrics likes/comments per post are bought, or the engagement series does not grow — -#: it costs a vendor record PER POST rather than per profile (measured: the -#: profile row carries post identity and no engagement) -#: commentMetrics the Comments dataset is bought, which can bill many rows per post -#: write DRY RUN — read everything, compute the counts, write nothing anywhere -#: -#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is -#: the point of the ruling: the money switches used to be "which rung do we try" (a question about -#: our plumbing), and they are now "what do you want captured" (a question about the user's data). -#: Profile is always captured and has no switch — an automation that fetches nothing is not an -#: automation with a step turned off, it is a broken one with a lie on it. -#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and -#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the -#: three toggles" names the CAPTURE ladder it is reshaping. -NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics", - "capture_comments": "commentMetrics", "write": "write"} - - -def _cron_label(cron): - return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") - - -def node_status(steps, nid): - """A node's dot — from the last run's MEASURED step outcome, or `idle`. - - ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely - so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, - and painting those nodes green because the run said `ok` would manufacture a measurement — the - same defect as the green dot over the empty posts table recorded in `run_field_instagram`. - No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. - """ - v = str((steps or {}).get(nid) or "") - return v if v in NODE_STATES else "idle" - - -def graph(defn): - """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" - defn = defn if isinstance(defn, dict) else {} - cfg = defn.get("config") or {} - sched = defn.get("schedule") or {} - runs = defn.get("runs") or [] - last = runs[0] if runs and isinstance(runs[0], dict) else {} - steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} - - def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): - return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, - "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), - "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} - - on = bool(sched.get("enabled")) - trg = defn.get("trigger") or {} - if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): - # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail - # read the same node, so an event-triggered flow must not read "Schedule". The membership - # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old - # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so - # the else-branch was reachable) — green, compiling, and wrong on screen. - t_on = bool(trg.get("enabled", True)) and not trg.get("paused") - sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) - watched = ", ".join(trg.get("fields") or []) or "any field" - # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the - # whole of it) and `record_updated` no longer names a condition (its watched fields are). - # These two lines are the migration's visible half: a node still describing a `.field` - # the validator has stopped storing would be the surface and the store disagreeing. - # ⚠ W34-T47 — A COLON, NOT A FULL STOP. R6's sweep turned these four em dashes into - # sentence breaks, which is right for a REFUSAL and wrong here: these are compact node - # SUBTITLES ("ut_leads: view v1"), not prose, and a full stop mid-label reads as two - # truncated fields rather than one qualified name. The rule the sweep encodes is "a - # sentence that used a dash usually wants a period"; a LABEL usually wants a colon. - det = {"event_field": (f"{trg.get('table', '')}" - + (f": {_lane_sentence(trg.get('when'))}" - if trg.get("when") else "")), - "record_updated": f"{trg.get('table', '')}: {watched}", - "record_created": str(trg.get("table") or ""), - "enters_view": f"{trg.get('table', '')}: view {trg.get('viewId') or 'unset'}", - "form_submitted": str(trg.get("table") or ""), - "webhook": "POST the hook URL to fire it", - # ⭐ WAVE 30 · T05 — BOTH corpus triggers, and the value is identical because the - # question is: a discovery trigger's detail line IS its filter. Without the TikTok - # key this fell to `.get(..., "")` and the trigger node rendered with a blank - # subtitle, which reads as "nothing configured" on a fully configured automation. - "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), - cfg.get("operator"))[:80] - if cfg.get("predicates") - else "No filters yet. Nothing to search for"), - "tiktok_profile_match": (_predicate_sentence(cfg.get("predicates"), - cfg.get("operator"))[:80] - if cfg.get("predicates") - else "No filters yet. Nothing to search for"), - "email": str(trg.get("query") or "")}.get(trg["key"], "") - if not trg.get("configured", True): - det = "Finish setting this trigger up before it can fire" - if trg.get("paused"): - det = str(defn.get("statusNote") or "paused") - nodes = [node("trigger", "trigger", sub, - "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] - else: - # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off - # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with - # the user's own pick — the same class of defect as the Database picker that could not be - # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the - # disabled-schedule gate asserts, and it was never the wrong half. - # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires - # TODAY, not that it may never be scheduled — the cron has to stay one click away, and - # this is the only node that offers it. - nodes = [node("trigger", "trigger", "Schedule" if on else "Manual", - _cron_label(sched.get("cron")) if on else "Manual only", 0, - panel="schedule", on=on, - detail="" if on else "It still runs when you press Run now")] - edges = [] - - if defn.get("kind") == "field_instagram": - # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED. - # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and - # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5 - # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the - # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners - # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas. - # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an - # implementation detail: the profile is always captured (R6), Posts and Comments are the - # opt-ins, and both are OFF until asked for. - metrics_on = bool(cfg.get("postMetrics")) - comments_on = bool(cfg.get("commentMetrics")) - max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL - nodes += [ - node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, - panel="source", - detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") - else "URL column: the one the field is bound to"), - # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY - # and no engagement (measured), so likes/comments are a SECOND vendor call per post. - # A switch that multiplies a run's cost by the post count deserves to be visible on - # the canvas rather than buried in a config panel. - node("capture_posts", "capture", "Post data", - "Likes and comments per post" if metrics_on else "Off", - 2, panel="capture", on=metrics_on, - detail=(f"Up to {max_posts} posts per profile" if metrics_on else - "ut_ig_post_snapshots only grows while this is on")), - node("capture_comments", "capture", "Comment data", - "Comments on those posts" if comments_on else "Off", - 3, panel="capture", on=comments_on, - detail=("The full comments dataset. Many rows per post" if comments_on else - "Comments already embedded in a paid post row are kept either way")), - node("write", "write", "Write", - cfg.get("targetTable") or "No database", 4, panel="write", - on=not cfg.get("dryRun"), - detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), - ] - edges = [{"from": "trigger", "to": "source", "label": ""}, - {"from": "source", "to": "capture_posts", "label": ""}, - {"from": "capture_posts", "to": "capture_comments", "label": ""}, - {"from": "capture_comments", "to": "write", "label": ""}] - elif defn.get("kind") in DISCOVERY_KINDS: - # ⭐⭐ WAVE 30 · T05 — WIDENED FROM `== "discover_instagram"`, AND THIS IS THE FAILURE T04 - # UNMASKS. With the seed fixed but this arm still Instagram-only, a stored TikTok search - # fell past every arm to the no-bare-`else` promise below and returned THE TRIGGER NODE - # ALONE — so no node carried `panel="find"`, and `AutomationFind` (which mounts on - # `panelKey === "find"`) could never appear. The person would have picked the trigger - # successfully and then found nowhere to type a filter: a second, stranger bug, arriving - # as the reward for fixing the first one. - _platform, _default_table, _, _ = discovery_facts(defn.get("kind")) - limit = int(cfg.get("recordsLimit") or 0) - est = discover_estimate(limit) - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - nodes += [ - node("find", "source", f"Find {_platform} profiles", - _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, - panel="find", - detail=(f"Up to {limit} profiles · about ${est['usd']}" - if bd_ready() - else "Profile search is not set up yet")), - node("collect", "capture", "Collect results", - "Searching…" if pending else "Takes about 20 minutes", 2, - panel="find", - detail=""), - node("write", "write", "Save results", - cfg.get("targetTable") or _default_table, 3, panel="write", - on=not cfg.get("dryRun"), - detail=""), - ] - edges = [{"from": "trigger", "to": "find", "label": ""}, - {"from": "find", "to": "collect", "label": ""}, - {"from": "collect", "to": "write", "label": ""}] - elif defn.get("kind") == "scrape_db": - url = str(cfg.get("url") or "") - host = (urlparse(url).hostname or url or "No URL") if url else "No URL" - which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" - else f"HTML table #{int(cfg.get('tableIndex') or 0)}") - mapped = len(cfg.get("fieldMap") or {}) - nodes += [ - node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), - node("extract", "capture", "Extract", which, 2, panel="columns", - detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " - f"{cfg.get('keyField') or ', '}"), - node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, - panel="write", on=not cfg.get("dryRun"), - detail=cfg.get("targetTable") or "a new database"), - ] - edges = [{"from": "trigger", "to": "fetch", "label": ""}, - {"from": "fetch", "to": "extract", "label": ""}, - {"from": "extract", "to": "write", "label": ""}] - # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point. - # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught - # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML - # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A - # surface that draws nothing is obviously incomplete; one that draws a fetch step that does - # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at - # once because all three read this. - # - # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE - # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node, - # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on - # that promise, so it is stated rather than left to be inferred from the control flow. - return {"nodes": nodes, "edges": edges} - - -def toggle_node(rt, auto_id, node_id): - """Flip ONE node's switch. Returns `(defn, error)`. - - The node → config mapping lives on the SERVER for the reason the whole graph does: a client - that knew which field each node writes would be a second copy of that knowledge, free to - drift. The canvas just says "this node was clicked". - """ - prev = all_definitions(rt).get(str(auto_id)) - if prev is None: - return None, "no such automation" - target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) - if target is None: - return None, f"there is no {node_id!r} step in this automation" - if not target.get("toggle"): - return None, (f"the {target['title']} step cannot be turned off. It is what this " - f"automation is") - cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) - which = target["toggle"] - trg_key = str((prev.get("trigger") or {}).get("key") or "") - if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS: - # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping - # the cron under a node labelled "When a field changes" would be a switch that lies. - # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that - # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so - # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so - # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half. - trg = dict(prev.get("trigger") or {}) - trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) - trg["paused"] = False # a deliberate flip clears an auto-pause: human wins - return patch(rt, auto_id, {"trigger": trg}) - if which == "schedule": - sched["enabled"] = not sched.get("enabled") - elif which == "postMetrics": - cfg["postMetrics"] = not cfg.get("postMetrics") - elif which == "commentMetrics": - cfg["commentMetrics"] = not cfg.get("commentMetrics") - elif which == "write": - cfg["dryRun"] = not cfg.get("dryRun") - return patch(rt, auto_id, {"config": cfg, "schedule": sched}) - - -# --------------------------------------------------------------------------------------------- -# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed -# --------------------------------------------------------------------------------------------- -# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were -# records of the target database, and each record's position lived in a single-select STAGE FIELD -# the engine created on the customer's own database. Wave 26 deleted the board's built-in -# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`, -# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review -# branch of the action walk, and the endings that sent a terminal card back round. -# -# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote -# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles` -# counter per automation — to render a view of state the RUN LOG already holds. The customer paid -# for that in columns they did not ask for, in a store commit on every scheduled run, and in a -# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we -# had added. `migrate_ig_tables` now drops those columns wherever an automation created them. -# -# ⛔⛔ `notify_review` IS GONE FROM HERE AND ITS CONSUMER IS STILL ALIVE — DEBT D-101, and this -# paragraph is its TOMBSTONE, placed where the next reader of the review machinery will look. -# -# `notify_review` was the producer of the `automation_review` notification: a card reaching a -# review stage told a human to come and look. It had ZERO callers even before R3 (wave-27 session -# C found it and correctly left it alone as another lane's subject), and R3's deletion took the -# function with the rest of the board. `verify_automation`'s W23-B section asserts its absence by -# name, so it cannot come back by accident. -# -# ⚠ THE SWEEP FOR ITS READERS, RUN 2026-08-12 (W31-T37) AND REPORTED RATHER THAN ASSUMED EMPTY: -# · SERVER — `notify_review` appears in exactly ONE file, `verify_automation.py`, inside the -# list of names asserted GONE. **No `.py` file anywhere produces an `automation_review` -# notification**; `core.alerts.notify` defaults `topic='automation'`, which is a different, -# live shape (the run-outcome alert). -# · CLIENT — the branch is FULLY ALIVE and is NOT in this fence: -# `web/src/alerts/alertsModel.ts` (`AUTOMATION_REVIEW_KIND`, `isAutomationReview`, the -# `autoId` guard) and `web/src/alerts/AlertsPane.tsx` (two call sites), plus legs in -# `web/verify_alerts.py`. All four are session B's files. -# ⇒ **The producer is deleted; the consumer guards an event that can never arrive.** D-101's own -# exit condition says these are one defect seen from two ends and must be closed in ONE change — -# so this half is the tombstone plus the sweep, and the client half is routed to B rather than -# reached across a fence. If a future wave gives review stages a real producer, THIS is the note -# that says the client already knows how to render it. -# -# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE: -# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the -# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was -# one of four callers. The name is the last thing the board left behind here. -# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board` -# — the MIGRATION. It must outlive the thing it retires, or a definition written before the -# retirement walks into a runtime that no longer understands it. -# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are -# parked with no caller and say so at their own definitions. - -LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", - "is_empty", "is_not_empty") -LANE_NULLARY_OPS = ("is_empty", "is_not_empty") - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` -# -# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why -# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches -# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc -# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the -# same function applied one level up, so the cheaper honest version is to evaluate the leaf where -# it lies and never touch the bytes.) -# -# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An -# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and -# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. -# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an -# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that -# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine -# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. -MAX_COND_DEPTH = 3 -MAX_COND_CHILDREN = 12 -COND_GROUP_KEYS = ("all", "any") - - -def clean_cond(raw, depth=0, where=""): - """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no - condition", which is legal everywhere a condition is optional (the catch-all lane, an - unconditioned trigger, an action group that always runs).""" - if raw in (None, "", {}): - return None, None - at = f" on {where}" if where else "" - if not isinstance(raw, dict): - return None, f"the condition{at} must be an object" - conj = [k for k in COND_GROUP_KEYS if k in raw] - if len(conj) > 1: - return None, (f"the condition{at} sets both 'all' and 'any'. A group is one or the " - f"other") - if conj: - key = conj[0] - if depth + 1 > MAX_COND_DEPTH: - return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep. " - f"the group{at} is deeper") - kids_raw = raw.get(key) - if not isinstance(kids_raw, list) or not kids_raw: - return None, f"the '{key}' group{at} needs at least one condition inside it" - if len(kids_raw) > MAX_COND_CHILDREN: - return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions. " - f"the '{key}' group{at} has {len(kids_raw)}") - kids = [] - for child in kids_raw: - c, err = clean_cond(child, depth + 1, where) - if err: - return None, err - if c is None: - return None, (f"an empty condition sits inside the '{key}' group{at}. " - f"finish it or remove it") - kids.append(c) - return {key: kids}, None - field = _s(raw.get("field"), 80).strip() - op = _s(raw.get("op") or raw.get("operator"), 20).strip() - if not field: - # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could - # act on. It states a fact about the stored tree; it never said what to DO, and the usual - # cause is not a typo but an empty field picker (the flow had no walking record to offer - # columns from). The fix belongs in the message. - return None, (f"the condition{at} names no field. Pick a column, or remove the " - f"condition") - if op not in LANE_OPS: - return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) - cond = {"field": field, "op": op} - if op not in LANE_NULLARY_OPS: - v = raw.get("value") - if v is None or (isinstance(v, str) and not v.strip()): - return None, f"give a value to compare {field!r} against{at}" - cond["value"] = v.strip() if isinstance(v, str) else v - return cond, None - - -def cond_fields(cond): - """Every field name a tree reads — the set a caller must have on the row image before the - answer means anything.""" - if not isinstance(cond, dict): - return set() - for key in COND_GROUP_KEYS: - if key in cond: - out = set() - for child in cond.get(key) or []: - out |= cond_fields(child) - return out - f = cond.get("field") - return {f} if f else set() -#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would -#: collide with the stage the label is a choice FOR, and the board could no longer tell a -#: routed card from a gated one. -# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the -# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a -# legacy stamped cell would read as the same lane while behaving as a different one. -def _lane_num(v): - try: - return float(str(v).replace(",", "")) - except (TypeError, ValueError): - return None - - -def lane_match(cond, row): - """Does `row` satisfy this condition TREE? None matches everything (the catch-all). - - Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 - caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping - this name rather than growing a second entry point that could disagree with it. - - ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse - as a number answers False — the record simply does not enter the lane — never "treat blank - as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank - cell is an unknown, and an unknown cannot be less than 10,000. - """ - if cond is None: - return True - if isinstance(cond, dict): - if "all" in cond: - return all(lane_match(c, row) for c in cond.get("all") or []) - if "any" in cond: - return any(lane_match(c, row) for c in cond.get("any") or []) - raw = (row or {}).get(cond.get("field")) - op = cond.get("op") - if op == "is_empty": - return raw in (None, "") - if op == "is_not_empty": - return raw not in (None, "") - want = cond.get("value") - if op in (">", ">=", "<", "<="): - a, b = _lane_num(raw), _lane_num(want) - if a is None or b is None: - return False - return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] - have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") - if op == "includes": - return want_s.lower() in have_s.lower() - if op == "not_includes": - return want_s.lower() not in have_s.lower() - a, b = _lane_num(raw), _lane_num(want) - same = (a is not None and b is not None and a == b) or have_s == want_s - return same if op == "=" else (not same) if op == "!=" else False - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. -# -# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. -# A `group` holds nested actions behind a condition — Airtable's "conditional action group", -# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's -# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this -# builder is a column, not a canvas). -# -# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a -# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / -# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty -# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready -# kind with a sentence, so the faded state is a wall and not a styling choice. -MAX_ACTIONS = 20 # per automation, counting nested ones -MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow -MAX_BRANCHES = 6 # per If / then -MAX_ACTION_VALUES = 20 # cells one update/create action may write -FIND_LIMIT_MAX = 100 -#: ⭐⭐ W31-T38 / E-4 — HOW MANY BROWSER JOBS ONE RUN MAY SUBMIT, and it belongs HERE because this -#: is the only layer that knows a record WALK is happening. `web_agent.MAX_STEPS` bounds steps per -#: JOB; nothing bounded jobs per RUN, and the web arm submits one job PER RECORD. A `web_read` on a -#: flow over the Customer grid is therefore thousands of ~10 s submissions against HF's -#: 6-concurrent cap — a run whose wall-clock is `records x 10 s` and a burst the `/hf-jobs` rules -#: exist to prevent. Session E raised it and could not fix it: the seam sees one step and cannot -#: know a walk is happening. -#: ⚠ 50 IS A STATED CHOICE, NOT A MEASUREMENT: at E's measured ~9-32 s per job that is roughly -#: 8-27 minutes of serial browsing, which is a long automation run and not a runaway one. It is -#: DISCLOSED when it binds (R6's second sentence — see the web arm), never silently applied. -#: ⛔ THE REAL FIX IS BATCHING, NOT A BIGGER NUMBER: `web_agent.run_plan(steps, ctx)` already takes -#: a list and the cost is per JOB, so a flow's web steps collected into ONE call would make this -#: ceiling nearly unreachable. That is E's own PENDING row; this bound is what stops the damage -#: until it lands. -MAX_WEB_JOBS_PER_RUN = 50 -#: ⭐⭐ W33-T58 (D-191) — HOW MANY WEB STEPS MAY SHARE ONE JOB. The comment above finally got its -#: batching, so the ceiling above now counts JOBS rather than pages and a record's consecutive web -#: steps cost one cold start between them instead of ~9 s each. -#: ⚠ 20 IS THE RUNNER'S OWN `MAX_STEPS` (`jobs/web_agent_job.py`), not a number chosen here: a plan -#: longer than that is refused by the job with `kind="too_many_steps"`, which would turn a -#: performance improvement into a whole record's worth of failed steps. Kept in step by -#: `verify_web_agent.py`, which reads both. -MAX_WEB_STEPS_PER_JOB = 20 -#: ⭐⭐ W33-T56 — HOW MANY STEPS ONE `ai_agent` ACTION MAY COMPOSE FOR ITSELF. -#: ⛔ A CEILING, NOT A TARGET, and it is the only bound between a sentence and a browser session. -#: The five `web_*` kinds are each ONE action a person wrote down; this one is a description that -#: the model turns into a journey at run time, so the number of real browser actions it performs is -#: decided by a paragraph rather than by the flow. Twelve is enough for "log in, search, open the -#: third result, read the price" and short of anything that reads as a program. -#: ⚠ Bounded by `MAX_WEB_STEPS_PER_JOB` as well, since the composed steps ride ONE job. -AI_AGENT_MAX_STEPS = 12 - - -#: The test seam for `_ai_agent_plan` — `None` in every shipped path. `verify_automation.py` sets it -#: so the fuzzy step is proven end to end with NO API key and NO spend, exactly as -#: `routes_automation._DRAFT_CHAT` does for the drafting door. -_AI_AGENT_CHAT = [None] - - -def _ai_agent_plan(cfg, row, log=print, st=None, user=""): - """A description + one record -> `(steps, "")` for `run_plan`, or `(None, sentence)`. - - ⭐⭐ W33-T56. This is the whole of the fuzzy step: the instruction and the record's own values - go to the model, and CONCRETE `web_*` steps come back — the same vocabulary a person could have - written by hand, so everything downstream (the seam, the job, the checkpoint, the per-step - verdicts) is unchanged and none of it has to know an assistant was involved. - - ⛔ IT MAY COMPOSE ONLY BROWSER STEPS. `ai_review.draft_flow` is handed a catalog filtered to - `WEB_KINDS`, so the enum in the tool schema cannot express `create_record` or a connector call. - A fuzzy sentence therefore cannot be talked into writing the tenant's data: the worst a bad - instruction can do is waste one browser session. That is a property of the SCHEMA, not of the - prompt, which is the only version of it worth relying on. - ⛔ AND IT IS BOUNDED. `maxSteps` is clamped at save time and re-applied here, because the - number of real browser actions is otherwise decided by a paragraph. - ⚠ `interpolate` FIRST, so the model is told about THIS record — `{{Website}}` is a different - page for every row, and a journey composed against the template would be one guess repeated. - """ - import ai_review # noqa: PLC0415 - url = interpolate(str(cfg.get("url") or ""), row) - instruction = interpolate(str(cfg.get("instruction") or ""), row) - cap = max(1, min(int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS), MAX_WEB_STEPS_PER_JOB)) - web_only = [r for r in ACTION_CATALOG if r.get("kind") in WEB_KINDS and r.get("ready")] - draft, why, _prov = ai_review.draft_flow( - prompt=(f"Starting page: {url}\n\nDo this: {instruction}\n\n" - f"Answer with at most {cap} steps. The FIRST step must open the starting page."), - catalog=web_only, - required={k: v for k, v in ACTION_REQUIRED.items() if k in WEB_KINDS}, - triggers=[], tables=[], - # ⭐ W35 · C7 (`NOTE E-16`) — threaded IN from `_walk` rather than resolved here: this - # function is pure over `(cfg, row)` by design and giving it a store handle of its own - # would be a second way to reach the tenant. - st=st, user=user, - chat=_AI_AGENT_CHAT[0]) - if why or not draft: - return None, (why or "the assistant produced no steps for that instruction") - steps, seen_url = [], False - for i, a in enumerate(draft.get("actions") or [], 1): - kind = str(a.get("kind") or "") - if kind not in WEB_KINDS: - # Belt and braces behind the enum: a rung that ignores its own schema is stopped here - # rather than reaching the browser. - return None, f"the assistant asked for a step this action cannot perform ({kind})" - c = a.get("config") or {} - step = {"kind": kind, "id": f"ai{i}", - "url": str(c.get("url") or "") or (url if not seen_url else ""), - "selector": str(c.get("selector") or ""), - "attr": str(c.get("attr") or "text"), - "all": bool(c.get("all")), - "waitFor": str(c.get("waitFor") or "") or None, - "timeoutMs": int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS)} - if c.get("value"): - step["value"] = str(c.get("value")) - if c.get("hint"): - step["hint"] = str(c.get("hint")) - if cfg.get("dryRun"): - step["dryRun"] = True - seen_url = seen_url or bool(step["url"]) - steps.append(step) - if len(steps) >= cap: - break - if not steps: - return None, "the assistant produced no steps for that instruction" - # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to act - # on yet — and the action's own `url` is exactly the answer. Supplying it here beats letting - # the whole job be refused for a sentence the model happened not to repeat. - if not steps[0].get("url"): - steps[0]["url"] = url - return steps, "" - - -def _web_missing(kind, cfg): - """Which of this kind's REQUIRED config keys are blank — the human phrases, in table order. - - ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and no selector; - `web_fill` needs a value nobody else takes; only `web_read` needs a column to write into. One - shared three-field test would have blocked every `web_goto` ever configured for want of a - selector it does not use. - ⭐ Lifted out of `apply_actions` at W33-T58 so the BATCH BUILDER and the per-step arm ask the - same question of the same table. Two copies of "is this step configured" would let a batch - include a step the arm then refuses — or worse, exclude one it would have run. - """ - return [n for n, key in WEB_REQUIRED.get(kind, ()) if not str((cfg or {}).get(key) or "").strip()] -#: ⭐⭐ W31 QA — THE FIVE WEB KINDS THE ENGINE DISPATCHES, and this is the THIRD list that names -#: them (`web_agent.RUNNABLE_KINDS` is the seam's, `jobs/web_agent_job.py::RUNNABLE_KINDS` is the -#: runner's). ⛔ IT IS DELIBERATELY NOT AN IMPORT: `_web_agent()` resolves the seam LAZILY so an -#: absent module cannot break the engine, and a module-level `from web_agent import RUNNABLE_KINDS` -#: would throw that property away for a tuple of five strings. The three lists are held in step by -#: `verify_web_agent.py::section_one_kind_only`, which reds if any pair disagrees — a per-kind -#: allow-list in three places that CAN disagree is the defect, whichever way it points. -WEB_KINDS = ("web_read", "web_goto", "web_fill", "web_click", "web_repair") -#: What each kind cannot run WITHOUT. Absence is never refused at SAVE (a step is addable before it -#: is configured — see `_clean_action_config`'s web arm); it is refused at RUN with a sentence -#: naming what is missing. `url` is required only by `web_goto`: every other kind acts on the page -#: the flow is already on, and the runner supplies its own refusal when it genuinely needs one. -WEB_REQUIRED = { - "web_read": (("a URL", "url"), ("a CSS selector", "selector"), - ("a column to write into", "field")), - "web_goto": (("a URL", "url"),), - "web_click": (("a CSS selector", "selector"),), - "web_repair": (("a CSS selector", "selector"),), - "web_fill": (("a CSS selector", "selector"), ("a value to type", "value")), - # ⭐⭐ W33-T56 — the fuzzy step needs a page to start on and a job to do, and nothing else. - # ⚠ NO `selector` AND NO `field`: working out the selector IS the step, and where to put the - # answer is optional (a journey may only need to have been performed). - "ai_agent": (("a URL", "url"), ("a description of what to do", "instruction")), -} -#: ⭐⭐ WAVE 32 · T45 (owner item 10) — WHAT EACH ACTION KIND CANNOT RUN WITHOUT, for every kind. -#: -#: `WEB_REQUIRED` above already WAS this table for five kinds, complete with the human phrase each -#: refusal says out loud, so this is its widening rather than a second opinion — spread in, never -#: re-typed, or a wave that adds a web key would teach the run refusal and not the label. -#: -#: ⛔ MOST KINDS ARE ABSENT, AND THE ABSENCES ARE THE INTERESTING PART. `create_record`, -#: `update_record`, `find_records` and `group` are REFUSED AT SAVE when their config is incomplete -#: (`_clean_action_config`: *"the create record action names no database"*, *"…writes no values"*, -#: *"a branch with no actions inside it does nothing"*), so a stored one is configured by -#: construction and a row here would be a second, weaker copy of a wall that already holds. -#: -#: ⛔⛔ AND `enrich_instagram` / `enrich_tiktok` ARE DELIBERATELY ABSENT DESPITE BEING THE OBVIOUS -#: CANDIDATE. Their `profileField` may be empty on purpose: the C3 profile FLAG on the target -#: database resolves the binding at run time (`profile_field_key`), so an empty one is a working -#: action on any flagged database and marking it Unconfigured would put a red label on the -#: commonest correct configuration there is. The genuinely unbound case still fails closed at RUN -#: with its own sentence — which is the honest division of labour: this table holds what can be -#: answered from the ACTION ALONE, and anything needing the target database's schema stays a -#: run-time refusal rather than becoming a store read on the automations LIST path (W30-T12 took -#: that read off this route; putting it back to draw a label would undo the wave before this one). -ACTION_REQUIRED = dict(WEB_REQUIRED) -#: ⭐ W31-T38 (C5) — the web step's own clamps. `20000` matches the default E's seam documents; -#: the ceiling exists because this blocks the record walk (~9-32 s of cold start ALREADY), and an -#: automation that can be configured to wait ten minutes per record is a stalled run, not a slow one. -WEB_READ_TIMEOUT_MS = 20000 -WEB_READ_TIMEOUT_MAX_MS = 120000 -#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder` -#: and never by a literal list of group names: a client-side ordering is a second copy of this -#: table, and the way it fails is that a group added here renders last, or not at all, with -#: nothing red anywhere. -ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} -ACTION_CATALOG = [ - # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ──────── - # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row - # is a WALL, not a styling choice — and each `detail` says the true reason rather than a - # placeholder, because "coming soon" on five rows is how a menu stops being believed. - # - # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24 - # scheduled this build for wave 25 and wave 25's six owner items do not include it, so - # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its - # own promises has to be re-read by whoever misses the date, and a stale date is worse than - # none because it reads as a commitment somebody already broke. The replacement names the - # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule. - # ⭐⭐ WAVE 31 · T38 + QA (C5) — ALL FIVE ROWS ARE READY, AND THE RULING IS WHY. - # - # ⛔ THIS REVERSES T38's ORIGINAL `how:`, ON THE OWNER'S OWN WORDS. T38 shipped `web_read` - # alone and held the other four at `ready:False`, citing PRD R10 ("the ones that WRITE to a - # third party do not flip without R5's approval gate existing"). **R5 was REVOKED the same - # day, and the revocation is recorded as an amendment on this wave's board** - # (`TICKETS.md:1418`, `mailbox/E.md:278`, `web_agent.py:103`) — owner, verbatim: - # *"make sure we unblock all web actions, we don't need approval step first wtf, I never ask - # for that."* Session C declined E's ask against the superseded half of R10, so four actions - # the owner asked for shipped unusable, and `verify_web_agent` was RED on correct-by-ruling - # code for the whole wave. Confirmed at QA against four independent recordings of the quote. - # ⚠ THE FADED ROW IS A WALL, NOT A STYLE: `clean_actions` refuses an unready kind with a - # sentence, so this flag is what makes the action storable at all — which is exactly why a - # row left `False` against a ruling is a feature that does not exist. - # ⚠ THE `detail` STRINGS CHANGED WITH THE FLAGS, deliberately. They said *"Needs the browser - # job"* and *"Needs the recorder that captures a selector"* — both false since E's runner - # landed. A menu that dates its own promises is this module's own recorded complaint one - # comment up; a row that advertises a missing prerequisite it already has is the same defect. - # ⭐⭐ WAVE 33 · W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP. - # ⛔ IT ADDS; IT REPLACES NOTHING. R3 is explicit: *"No existing `web_*` kind is removed — - # dropping a kind from the catalog 400s every stored automation using it, forever"* (D-65). - # This is the step for a journey somebody can DESCRIBE but not spell as a selector; the five - # kinds below stay exactly as they are for the journeys they can already express, and a person - # who knows the selector should still use `web_read`, which costs no model call. - # ⭐⭐ WAVE 34 · W34-T44 / R18 — SIX ROWS BECAME ONE, AND THE OTHER FIVE DID NOT LEAVE THE - # CATALOG. Owner, verbatim: *"Remove Do this on a page / Open a page / Fill a field / Click - # something / Read from the page / Repair a broken step completely. One action, Web agent."* - # - # ⛔⛔ WHY THEY ARE HIDDEN AND NOT DELETED, AND IT IS NOT D-65 THIS TIME — IT IS WORSE. - # `_ai_agent_plan` builds the model's tool schema as - # `[r for r in ACTION_CATALOG if r["kind"] in WEB_KINDS and r["ready"]]` - # so DELETING these five rows empties that list, and the ONE action the ruling keeps would be - # left able to compose nothing at all. Deleting the six would have deleted the survivor, in - # silence, and `ai_agent`'s own tests would still pass because they stub the chat rung - # (`_AI_AGENT_CHAT`). D-65's usual argument (a deleted kind 400s every stored automation - # forever) applies as well and is the smaller half. - # - # ⭐ SO THE LINE MOVED FROM "IS IT IN THE CATALOG" TO "IS IT ON THE MENU". `menu: False` is - # withheld by `action_catalog()` from the picker, while `clean_actions` (which reads this - # constant, not the wire) still validates the kind, the runner still runs it, and a stored - # automation built before today keeps working and stays editable. - # ⚠ THE LABELS CHANGED TOO, and that is R18's "appear NOWHERE" clause taken literally: a - # hidden row's label is still rendered on a STORED step's card, so leaving the six captions in - # place would have kept them on screen for exactly the people who already use them. They now - # name the mechanism instead, and no old caption survives as a substring in any case. - {"kind": "ai_agent", "label": "Web agent", "group": "Web action", "ready": True, - "detail": "Describe a job on a website in words; the agent works out the steps, runs them in " - "a browser and reports what it actually did"}, - {"kind": "web_goto", "label": "Web agent step (navigate)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Opens a page in a browser job and reports the title it landed on"}, - {"kind": "web_fill", "label": "Web agent step (type)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Types a value into a field. Mark it secret and the value is masked in the log"}, - {"kind": "web_click", "label": "Web agent step (click)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Clicks the element a selector names, and reports where it landed"}, - # ⚠ `"kind": "web_read"` AND `"ready": True` STAY ON ONE LINE. `verify_wiring`'s C5 row matches - # `"kind": "web_read".*?"ready": True` without DOTALL, so wrapping this row the way its four - # siblings are wrapped turns that cross-fence assertion red — on a formatting change, with the - # mount and the flag both intact. The row is A's file and its CLAIM is right (a catalog kind - # the client can add must be one the runner will execute); the layout is what it happens to - # depend on, so the layout is preserved here rather than the assertion weakened there. - {"kind": "web_read", "ready": True, "label": "Web agent step (extract)", - "group": "Web action", "menu": False, - "detail": "Reads one value off a live page in a browser job. Expect ~10-30 s per step"}, - {"kind": "web_repair", "label": "Web agent step (relocate)", "group": "Web action", - "ready": True, "menu": False, - "detail": "Follows a label to its control when a selector has gone stale, and proposes one"}, - # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── - {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, - "detail": "Write values onto the record walking the flow"}, - {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True, - "detail": "Add a row to another database"}, - {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True, - "detail": "Look rows up by condition; the run log opens them"}, - # ── 3. CONNECTED ───────────────────────────────────────────────────────────────────────── - # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND. - # `field_instagram` was an automation you created to fill one column; this is a step any flow - # can take, which is the shape it should always have had — enriching a profile is something - # you do TO a record, not a species of automation. - # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu, - # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`. - {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected", - "ready": True, "connector": "scraper", - "detail": "Fill this record's Instagram columns from its profile, and add a point to its " - "history"}, - # ⭐⭐ WAVE 30 · T08 / CONTRACT C3 — THE SECOND NETWORK, AND THIS ROW IS THE LAST SWITCH THAT - # LANDS, deliberately. `apply_actions._walk`'s kind dispatch has NO terminal `else` (measured - # from the AST, not read): an unknown kind is walked, counted, reports the run `ok`, and - # writes nothing. So a catalog entry ahead of its runner arm would ship an action that is - # addable, clickable, storable and silently inert — strictly worse than the state before it, - # where `clean_actions` refuses the kind with a sentence. - # ⚠ `connector: "scraper"` NESTS IT UNDER THE SAME BUCKET AS INSTAGRAM (R3), which is the same - # value T07 gave the trigger — one word, both menus. - {"kind": "enrich_tiktok", "label": "Enrich TikTok profile", "group": "Connected", - "ready": True, "connector": "scraper", - "detail": "Fill this record's TikTok columns from its profile, and add a point to its " - "history"}, - {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, - "detail": "Needs a send scope on the Gmail connection"}, - # ⭐⭐ WAVE 35 · T35 / CONTRACT C8 / OWNER RULING R10 — STATEMENTS BECOME AN AGENT STEP. - # - # Owner item 14: move Statements out of Settings and into the agent automation, "templatic and - # easily toggleable". R10 is the half that decides the shape: the step ASSEMBLES and PARKS a - # batch in a review stage, and **nothing sends without a human click**. So this row's promise - # is deliberately "prepares", not "sends" — the label a person reads must not describe an act - # the step does not perform. - # - # ⛔ ADDED BESIDE `send_email`, NEVER BY REPURPOSING IT (the ticket's own trap, D-295): a - # stored automation naming a kind that no longer exists is refused FOREVER rather than dropped - # with a reason, so adding a kind is cheap and re-pointing one is not. - # - # ⛔ `ready: True` IS WHAT MAKES IT STORABLE — `clean_actions` refuses any row whose `ready` is - # false — and the note at `enrich_tiktok` above is the reason the RUNNER ARM lands in the same - # wave rather than after it: `_walk` has no terminal `else`, so a catalog row ahead of its arm - # is addable, storable and SILENTLY INERT, which is worse than not existing. T35 lands an arm - # that reports it is not configured; T36 makes it park a real batch. - # - # ⚠ TENANT-GATED, not `ready: False`: `TENANT_GATED_ACTIONS` withholds this row from every - # tenant but #0, because the send client behind it is env-credentialed and is tenant #0's. - {"kind": "send_statement", "label": "Prepare customer statements", "group": "Connected", - "ready": True, - "detail": "Assemble this month's statements and park them for review. Nothing is sent until " - "somebody opens the batch and clicks Send"}, - # ⭐⭐ WAVE 36 · W36-T39 — D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an - # `ACTION_CATALOG` row with `menu: false` — one line, reusing the door R18 built this same wave - # to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."* - # - # ⛔ THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows` - # emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and - # that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The - # builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls - # back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN. - # - # ⚠ `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" — a promise — - # and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment - # anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker, - # still resolvable as a caption, still valid. - # ⛔ AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation - # canvas is not where one is created, which is why `patch_automation` already refuses a - # `field:` id with "change its prompt, model or schedule on the column itself". - # ⭐⭐ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row` - # emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and - # that kind was in no catalog either — the same raw token on the same screen as D-277, one row - # down. Two known instances were enough to justify the check; the check then produced a third - # nobody had booked, which is the difference between a gate and a regression test. - {"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected", - "ready": True, "menu": False, - "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not " - "here"}, - {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected", - "ready": True, "menu": False, - "detail": "Fill an AI column for the records this flow walks. Configured on the column, not " - "here"}, - {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, - "detail": "Needs the Slack connector"}, - # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── - # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism, - # and R8 made it a FORK with lettered branches, which is a thing people already have a name - # for. The KIND is untouched — renaming it would orphan every stored action for a caption. - {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True, - "detail": "Send the record down one of several branches, by condition"}, - {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", - "ready": False, "detail": "Run the same actions on every item in a list"}, - {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, - "detail": "Not built. A sandbox is its own decision"}, - {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", - "ready": False, "detail": "Needs an AI action implementation"}, -] - - -#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can -#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an -#: "Integrations" header, one row per connector, a chevron into that connector's own submenu). -#: -#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are -#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the -#: key and render the label, never join it against the directory. -#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key. -#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one -#: connector wearing two names on two screens somebody sees within a second of each other, and a -#: second literal is how that happens. This tuple says only WHICH connectors an action may name; -#: what they are CALLED has exactly one source. -ACTION_CONNECTORS = ("scraper",) - - -def _connector_meta(key): - """`{key, label}` for a connector key, from the one place either picker declares it.""" - key = str(key or "") - if key not in ACTION_CONNECTORS: - return None - return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) - - -#: ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TENANT PREDICATE `routes_statements._royal_only` USES, -#: LIFTED SO THERE IS EXACTLY ONE COPY OF IT. C8's words are "the predicate is imported, never -#: re-expressed", and this is the direction that import can run: `automation_engine` imports no -#: route module and no FastAPI (checked), and breaking that to reach `_royal_only` would drag -#: `deps` + `routes_admin` into the engine. So the ENGINE holds the test and the ROUTE calls it. -#: -#: ⛔⛔ THIS IS NOT `odoo_relational.is_royal`, AND THE DIFFERENCE IS A SEND. That one asks "is this -#: tenant ENTITLED to Odoo databases" over `RI_SLUGS = ("", "royal-imports")` — it answers TRUE for -#: the EMPTY slug and lower-cases its input. This one is `_royal_only`'s exact test: the runtime's -#: own `key`, matched exactly. A tenant whose key never got set would pass `is_royal("")` and then -#: queue statements THROUGH TENANT #0'S ENV-CREDENTIALED SEND CLIENT, i.e. email another company's -#: customers over Royal Imports' name. Entitlement to READ is not authority to SEND, and the two -#: questions keep their two predicates on purpose. Do not "unify" them. -ROYAL_TENANT_KEY = "royal-imports" - - -def is_statement_tenant(rt): - """Exactly `routes_statements._royal_only`'s test, as a boolean over the RUNTIME. - - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION, so a - payload cannot argue its way into another company's sender. - """ - return getattr(rt, "key", None) == ROYAL_TENANT_KEY - - -#: Kinds only SOME tenants may see or store, as `{kind: predicate(rt) -> bool}`. -#: -#: ⛔ FAIL-CLOSED ON `rt=None`, AND THAT IS THE WHOLE SAFETY ARGUMENT. Every reader below treats an -#: absent runtime as "not allowed", so a call site that forgets to pass one makes the action VANISH -#: rather than become universal. The opposite default would mean any future caller of -#: `action_catalog()` or `clean_actions()` silently offers tenant #0's send door to every tenant — -#: an omission that is invisible in review and loud only in production [[default-must-pass-its-own-guard]]. -TENANT_GATED_ACTIONS = {"send_statement": is_statement_tenant} - -#: The dunning buckets a statements step may filter on. ⚠ ONE LITERAL, TWO READERS: this and -#: `routes_statements.statements()`'s `"tiers"` field are the same four strings, and a step -#: configured against a tier the sender's worklist does not produce would filter to nothing and -#: report success. `routes_statements` imports this rather than repeating it. -STATEMENT_TIERS = ("A-Urgent", "B-Active", "C-Light", "Monitor") - - -def _tenant_may_use(kind, rt): - """May this runtime see/store this action kind? Ungated kinds are always yes.""" - gate = TENANT_GATED_ACTIONS.get(str(kind or "")) - return True if gate is None else bool(rt is not None and gate(rt)) - - -def catalog_kinds(): - """Every action kind this module knows about — the LABEL vocabulary. - - ⭐ D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the - catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog - does not carry is a token on somebody's screen. This is the set that must cover every kind any - server path can put into a `flow.actions` entry, whether or not a person may add it. - """ - return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG) - - -def configurable_kinds(): - """The kinds a PERSON may add from the picker, and must therefore be able to configure. - - ⭐⭐ W36-T39 — THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is - the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is - ready and `menu: False` (D-277) — all six are real, runnable, captioned, and unofferable. What - a client must be able to CONFIGURE is exactly what a person can ADD. - - ⛔ DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list - would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile - were already one too many. - """ - return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG - if row.get("ready") and row.get("menu", True) is not False) - - -def action_catalog(rt=None): - """The catalog as the wire carries it — a copy, because a caller that mutated the module - constant would change every later reader's answer. - - ⭐⭐ WAVE 35 · T35 (C8): `rt` filters TENANT-GATED rows out entirely — not `ready: False`, not - `menu: False`, but ABSENT. A tenant that may not send statements should not learn that the - capability exists, and `ready: False` renders as "coming soon", which is a promise we are not - making to them. ⚠ Called with no `rt` the gated rows are withheld (fail-closed): see - `TENANT_GATED_ACTIONS`. - - ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from - `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two - different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST - (not first) — a new group appearing above "Web action" because its order defaulted to 0 is - the failure that would look deliberate. - - ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client - nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two - rows in one nest cannot disagree about what that nest is called. A row naming an unknown - connector loses the stamp instead of inventing a nest with a raw slug for a title. - - ⭐⭐ WAVE 34 · W34-T44 / R18: `menu` is stamped on EVERY row, never left absent. A row the - picker must not offer carries `menu: False` and STILL RIDES THE WIRE, because the client - resolves a STORED step's label out of this same list (`AutomationBuilder`: - `catalog.find(c => c.kind === a.kind)` then `row?.label || a.kind`) — withholding the row - entirely would make an existing web step render its raw kind token at somebody, which this - module forbids in those words elsewhere. - ⚠ STAMPED RATHER THAN LEFT TO DEFAULT: absent-means-true is a rule two codebases have to - remember the same way, and a client filter is one `!== false` away from meaning the opposite. - An explicit boolean on every row cannot be read two ways. - """ - last = max(ACTION_GROUP_ORDER.values()) + 1 - out = [] - for a in ACTION_CATALOG: - if not _tenant_may_use(a.get("kind"), rt): - continue - row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last), - "menu": bool(a.get("menu", True))} - conn = _connector_meta(a.get("connector")) - if conn: - row["connector"] = dict(conn) - else: - row.pop("connector", None) - out.append(row) - return out - - -def action_needs(action): - """⭐⭐ WAVE 32 · T45 (owner item 10) — what THIS action is still missing, in the words a person - reads. `[]` means Configured. - - Pure over `(kind, config)` — no runtime, no store read. That is what lets `_wire` stamp every - action on the automations LIST without putting the `user_tables` document back on a route - W30-T12 just took it off. - """ - action = action or {} - cfg = action.get("config") if isinstance(action.get("config"), dict) else {} - return [name for name, key in ACTION_REQUIRED.get(str(action.get("kind") or ""), ()) - if not str(cfg.get(key) or "").strip()] - - -def _walk_actions(actions): - """Every action in a flow, INCLUDING the ones nested inside If / then branches. - - ⛔ A FLAT `for a in actions` MISSES HALF A FLOW. `group.config.branches[].actions` is where a - conditional puts its real work, and a configured-check that only saw the top level would - report a flow ready to run while the step inside branch B had never been filled in — which is - the exact class of defect this ticket exists to surface, hiding inside the ticket's own fix. - """ - for action in actions or []: - if not isinstance(action, dict): - continue - yield action - for branch in ((action.get("config") or {}).get("branches") or []): - if isinstance(branch, dict): - yield from _walk_actions(branch.get("actions")) - - -def unconfigured_actions(defn): - """Every ENABLED action of this automation that cannot run, as `[{id, kind, label, needs}]`. - - ⚠ DISABLED ACTIONS ARE SKIPPED, and that is the point of being able to disable one: a step - somebody switched off is not a step blocking the run. `apply_actions` already ignores them. - """ - out = [] - for action in _walk_actions(((defn or {}).get("flow") or {}).get("actions")): - if not action.get("enabled", True): - continue - needs = action_needs(action) - if needs: - out.append({"id": str(action.get("id") or ""), "kind": str(action.get("kind") or ""), - "label": _action_label(action), "needs": needs}) - return out - - -def run_refusal(defn): - """⛔ WAVE 32 · T45 — the sentence a run is refused with, or `""`. - - Owner item 10: *"running an automation with ANY unconfigured action is REFUSED with a message - naming which action"*. Naming it is half the requirement and the half that is easy to drop — a - bare *"an action is not configured"* on a twelve-step flow is a hunt, not a message. - - ⛔ IT LIVES ON THE ENGINE, NOT ON THE ROUTE, BECAUSE THE ROUTE IS NOT THE ONLY DOOR. The tick - runs automations on a schedule, the webhook door runs them, and a check mounted in - `POST /automations/{id}/run` alone would refuse the button and let the cron sail past it — - [[seal-the-transport-not-the-rung]], and D-112's own shape (an action bound to a deleted view - walked zero records and reported `ok`). `run_now` refuses for every caller; the route asks - first only so the person clicking gets a 400 with the sentence instead of a silent no-op. - - ⚠ THIS IS A BEHAVIOUR CHANGE FOR STORED AUTOMATIONS AND IT IS THE RULING. A flow with one - unconfigured web step used to run, skip that step with a note, and report `ok`; it now does not - run at all. That is what "blocks the run" means, and the alternative — running everything else - and reporting success — is precisely what the owner is asking to stop. - """ - missing = unconfigured_actions(defn) - if not missing: - return "" - return "; ".join(f"{m['label']} still needs " + ", ".join(m["needs"]) for m in missing) - - -def _action_label(act): - for row in ACTION_CATALOG: - if row["kind"] == act.get("kind"): - return row["label"] - return str(act.get("kind") or "Action") - - -def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None, rt=None): - """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. - - ⭐⭐ WAVE 35 · T35 (C8) — `rt` IS THE TENANT WALL ON THE STORE SIDE, and it is a separate wall - from `action_catalog(rt)`'s. Withholding a row from the MENU stops it being offered; it does - not stop a hand-written body naming the kind, and the picker is not a security boundary - [[opening-a-route-widens-every-field]]. ⚠ Keyword-only in effect and defaulting to None, so - every existing caller is untouched by construction — the same shape `notes` used, for the - reason this docstring already gives about a signature change failing at RUN, not at import. - ⛔ `rt=None` REFUSES a gated kind rather than allowing it (`TENANT_GATED_ACTIONS`). - - Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder - survives a Save. A missing or colliding one is minted `act_`; minting on every clean would - move the selection under the person editing it. - - ⭐⭐ D-75 — `notes` IS THE DISCLOSURE CHANNEL, AND IT IS OPT-IN. Pass a list and this function - appends one plain sentence per thing it SILENTLY CHANGED: a `create_record` condition dropped - (see the note at that branch), and any config key an arm's allowlist did not keep. - - ⛔ WHY IT IS AN OUT-PARAMETER RATHER THAN A THIRD RETURN VALUE. D-75's own exit says *"this is - a signature change across its callers"* and that is exactly what makes the obvious fix - dangerous mid-wave: `clean_actions` is called from `clean_flow`, from `_is_untouched_ig_seed` - and recursively from its own group arm, and a caller that unpacked two values from a - three-tuple fails at RUN, not at import. An optional list defaults to `None`, every existing - caller is untouched by construction, and the one door that wants to tell somebody opts in. - ⚠ IT REPORTS, IT NEVER REFUSES. Every drop here is deliberate and D-65 is the reason — refusing - a stored automation's condition would 400 it forever, with no way to edit it out, because the - editor cannot save the automation it needs to fix. A drop is recoverable; a locked door is not. - What was missing was never the refusal, it was somebody being told. - """ - if raw in (None, ""): - return [], None - if not isinstance(raw, list): - return None, "actions must be a list" - seen = _seen if _seen is not None else set() - count = _count if _count is not None else [0] - out = [] - for entry in raw: - if not isinstance(entry, dict): - return None, "each action must be an object with a kind" - kind = _s(entry.get("kind"), 30).strip() - row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) - if row is None: - return None, (f"{kind or 'that action'!r} is not one of: " - + ", ".join(a["kind"] for a in ACTION_CATALOG)) - if not row["ready"]: - return None, (f"{row['label']!r} is on the menu but not built yet. " - f"{row['detail'][0].lower()}{row['detail'][1:]}") - # ⭐⭐ W35-T35 (C8) — THE TENANT WALL. Refused, not dropped, and this is the one place in - # this function where a refusal is right: D-65's "drop, never refuse" protects a stored - # automation from becoming permanently unsavable, and NO tenant this can refuse has ever - # been able to store one — the kind is withheld from their catalog, so there is no legacy - # body to strand. Dropping it silently would instead let a flow save, look saved, and never - # do the step the person configured. - if not _tenant_may_use(kind, rt): - return None, (f"{row['label']!r} is not available in this workspace. It sends as " - f"Royal Imports, using Royal Imports' own mail credentials") - count[0] += 1 - if count[0] > MAX_ACTIONS: - return None, f"an automation runs at most {MAX_ACTIONS} actions" - aid = _s(entry.get("id"), 40).strip() - if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: - n = 1 - while f"act_{n}" in seen: - n += 1 - aid = f"act_{n}" - seen.add(aid) - # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER. - # - # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't - # think you can even add Conditions at all. That's not how the Create Record works."* - # Airtable agrees, and so does the shape: every other action operates ON the record the - # flow is walking, so "run this only when …" is a question about something - # that exists. Create record MAKES one. There is nothing to test yet, which is why the - # picker on this panel was empty and why every condition saved against it named no field — - # the owner's *"the condition on action act_1 names no field"*. The condition belongs on - # the trigger, or on a conditional group wrapping the action. - # - # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored - # automation that already carries one — forever, with no way to edit it out, because the - # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a - # locked door. - # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26 - # removes the condition editor from this action kind entirely (C4), so there is no control - # whose value could appear to be ignored. If a `create_record` condition editor ever comes - # back, this needs a disclosure channel — `clean_actions` has none today. - if kind == "create_record": - # D-75: the drop is unchanged; what is new is that a caller can be TOLD about it. - if notes is not None and entry.get("when"): - notes.append(f"{aid}: the condition on a Create-record step was dropped. A " - f"Create record has no record to test yet. Put the condition on the " - f"trigger, or on an If wrapping this step.") - when = None - else: - when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") - if cerr: - return None, cerr - cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} - cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count, notes=notes, rt=rt) - if cerr: - return None, cerr - # ⛔ D-75 — THE ALLOWLIST'S OWN DROPS, DIFFED HERE RATHER THAN REPORTED BY EACH ARM. Every - # arm builds a fresh `out = {...}`, so none of them knows what it did not keep; the caller - # of the arm does, because it holds both dicts. One diff, and a new arm is covered the day - # it is written rather than the day somebody remembers to instrument it. - if notes is not None and isinstance(cfg, dict): - gone = [k for k in cfg_raw if k not in cfg] - if gone: - notes.append(f"{aid}: {', '.join(sorted(gone))} " - f"{'is' if len(gone) == 1 else 'are'} not a setting a " - f"{KIND_LABELS.get(kind, kind)!r} step uses, so " - f"{'it was' if len(gone) == 1 else 'they were'} not saved.") - out.append({"id": aid, "kind": kind, - "enabled": bool(entry.get("enabled", True)), - "when": when, "config": cfg}) - return out, None - - -def _clean_action_config(kind, cfg, depth, seen, count, notes=None, rt=None): - """One action's `config`, per kind. Returns `(config, error)`. - - ⚠ `rt` is threaded ONLY so the `group` arm can hand it back to `clean_actions` for the branch - recursion (W35-T35 / C8). Without it a tenant-gated kind is refused at the top level and - ACCEPTED inside an If, which is the half of a flow `_walk_actions`' own docstring says people - cannot see. - """ - if kind == "group": - # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, - # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an - # if with no else, so expressing "otherwise" meant a second group carrying the negation - # by hand — two conditions a later edit could put out of step with each other. - if depth + 1 > MAX_GROUP_DEPTH: - return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep. " - f"past that a flow is a flowchart and belongs on the board") - raw_branches = cfg.get("branches") - if not isinstance(raw_branches, list) or not raw_branches: - # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and - # the live automations carry it — refusing it would 400 every Save of a flow that - # was legal yesterday. Read-side, so nothing has to be rewritten in the store. - raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), - "actions": cfg.get("actions")}] - if len(raw_branches) > MAX_BRANCHES: - return None, (f"an If / then has at most {MAX_BRANCHES} branches. Past that the " - f"record is being routed, which is what the board's lanes are for") - out_branches, bseen = [], set() - for i, br in enumerate(raw_branches): - if not isinstance(br, dict): - return None, "each branch of an If / then is an object" - cond, cerr = clean_cond(br.get("cond"), where="the branch") - if cerr: - return None, cerr - if cond is None and i != len(raw_branches) - 1: - # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches - # and a null condition matches everything, so a catch-all above another branch - # would make every branch below it dead code — silently, and only for the - # records that reached it. Refused with the reason rather than reordered: this - # module does not quietly rewrite what somebody built. - return None, ("only the LAST branch of an If / then can be the Otherwise leg. " - "a catch-all above another branch makes the ones below it " - "unreachable") - # D-75: threaded into the branch too — an unconfigured step inside an If is - # exactly the one a person cannot see, which is `_walk_actions`' own argument. - kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count, - notes=notes, rt=rt) - if kerr: - return None, kerr - if not kids: - return None, "a branch with no actions inside it does nothing" - bid = _s(br.get("id"), 40).strip() - if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen: - bid = f"b{i + 1}" - bseen.add(bid) - label = " ".join(_s(br.get("label"), 40).split()) or ( - "Otherwise" if cond is None else _branch_letter(i)) - out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids}) - return {"branches": out_branches}, None - if kind in ("update_record", "create_record"): - values = cfg.get("values") - if not isinstance(values, dict) or not values: - return None, f"the {kind.replace('_', ' ')} action writes no values" - if len(values) > MAX_ACTION_VALUES: - return None, f"one action writes at most {MAX_ACTION_VALUES} cells" - clean_vals = {} - for k, v in values.items(): - key = _s(k, 80).strip() - if not key: - return None, "a value is written to a field with no name" - if v is not None and not isinstance(v, (str, int, float, bool)): - return None, (f"the value for {key!r} must be text or a number. " - f"an automation writes cells, not objects") - clean_vals[key] = _s(v, 500) if isinstance(v, str) else v - out = {"values": clean_vals} - # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is - # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic - # "Create record". Follows the `review` arm below, which has carried `config.label` since - # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config` - # bag so no shared client type has to change for it. - lbl = " ".join(_s(cfg.get("label"), 60).split()) - if lbl: - out["label"] = lbl - if kind == "create_record": - table = _s(cfg.get("table"), 60).strip() - if not table: - return None, "the create record action names no database" - if not table.startswith(UT_PREFIX): - return None, (f"actions write to blank databases (ut_*) this wave. " - f"{table!r} is not one") - out["table"] = table - # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS - # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a - # Create record duplicates a row per run — silently, and worse every day. - # - # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a - # guess: a stored automation must not change meaning because this key arrived. The - # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`. - # - # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the - # action never sets means every incoming row carries a BLANK key — `upsert_rows` - # counts those as `skipped` and writes nothing at all. That is a control that reads - # as "no duplicates" and delivers "no rows", which is the worse failure by a distance. - # Same shape as `scrape_db`'s "the key field must be one of the mapped fields" - # (`clean_config`, above) — one precedent, not a second mechanism. - unique = _s(cfg.get("uniqueOn"), 80).strip() - if unique and unique not in clean_vals: - return None, (f"this action does not write {unique!r}, so it cannot keep records " - f"unique on it. It writes: " + ", ".join(sorted(clean_vals))) - out["uniqueOn"] = unique - return out, None - if kind == "send_statement": - # ⭐⭐ WAVE 35 · T35 / R10 — WHAT A STATEMENTS STEP IS CONFIGURED WITH: a customer filter - # and a template. Both are the SENDER's own vocabulary (`routes_statements.send` passes - # `templates: {subject, intro, footer}` straight to `collections_send.queue_statement`), - # so this arm names no field the send door does not already take. - # - # ⚠ EVERY TEMPLATE FIELD IS OPTIONAL AND EMPTY MEANS "THE DEFAULT", which is exactly how - # the send door already reads them (`t.get("subject") or cs.DEFAULT_SUBJECT`). Storing our - # own copy of the default instead would freeze today's wording into every stored agent and - # silently stop tracking `collections_send`'s. - out = {} - tier = _s(cfg.get("tier"), 40).strip() - # ⛔ REFUSED, NOT COERCED, AND THE ANSWER LISTS THE REAL ONES — the same shape the connector - # cadence uses. A tier nobody sends to would filter the batch to zero customers and report - # a successful run: a silent no-op is the worst outcome available here, because the person - # believes their customers were invoiced. - # ⚠ `""` IS LEGAL and means EVERY tier. It is the stored default, so an agent saved before - # anybody picks a filter does not change meaning the day this key arrives (D-65's rule). - if tier and tier not in STATEMENT_TIERS: - return None, (f"{tier!r} is not a collection tier. Pick one of: " - + ", ".join(STATEMENT_TIERS) + ", or leave it blank for all of them") - out["tier"] = tier - for key, cap in (("subject", 200), ("intro", 4000), ("footer", 4000)): - out[key] = _s(cfg.get(key), cap) - lbl = " ".join(_s(cfg.get("label"), 60).split()) - if lbl: - out["label"] = lbl - return out, None - if kind in ENRICH_KINDS: - # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the - # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for - # TikTok would drift until one network accepted a limit the other refused. - # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid - # rung, its fallback, the per-post engagement buy and the dry run — because they are the - # things that decide what a run COSTS and R4 moves the capability, not the controls. - # - # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than - # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a - # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a - # sentence at run (`profile_field_key`), which is the honest half — refusing the save - # would make the action unaddable on a database that has not been flagged yet. - # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent - # clamps on the same key is how the action and the legacy kind come to disagree about - # what "10 posts" means. - # - # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference - # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH, - # so a key being there is a person having typed it. This function gets the whole action - # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a - # `maxPosts` here is usually just the client echoing back what was already stored — and - # every action written before this wave stored the old default of 24. Refusing would - # therefore 400 those automations forever on a number nobody chose, which is D-65. - # The ceiling is still taught, in the place where a person can act on it: the panel's - # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more). - max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False) - if perr: - return None, perr - # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a - # working default. That is D-65's lesson applied ahead of time rather than after: an enrich - # action stored before this wave carries none of these, and it must keep saving and keep - # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT; - # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching. - # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door - # sees a person's typed number and can refuse politely, while the selection sees whatever - # is STORED — including configs written before the cap existed. Clamping only here would - # let an old 5000 through at run time. - try: - e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) - except (TypeError, ValueError): - return None, "how many records to enrich has to be a number" - if e_limit < 1: - return None, "an enrich step has to be allowed at least one record" - try: - e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) - except (TypeError, ValueError): - return None, "the number of days has to be a number" - groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts) - if gerr: - return None, gerr - return {"profileField": _s(cfg.get("profileField"), 80).strip(), - "fromView": _s(cfg.get("fromView"), 80).strip(), - "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT, - # Anything that is not "asc" is newest-first, so a typo cannot invert the order a - # person is spending against — it lands on the documented default instead. - "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc" - else "desc", - "limit": min(e_limit, MAX_ENRICH_PER_RUN), - "skipRecent": bool(cfg.get("skipRecent")), - "skipRecentDays": max(1, e_days), - # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in - # `clean_config` (R5 / C2): a stored action carrying them still saves and simply - # loses them, because refusing a retired key 400s every definition written before - # this wave (D-65). - # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for. - # ⭐ WAVE 30 · T10 — THE TIKTOK FORCE-OFF IS GONE, because the capability it was - # standing in for now exists. T08 shipped `... and kind != "enrich_tiktok"` here - # deliberately: the key was ACCEPTED (never a 400, D-65's lesson) and answered - # honestly with `False`, because storing a `True` nothing acts on is a money switch - # claiming a capability nobody built. Both networks now read the SAME two keys. - "postMetrics": bool(cfg.get("postMetrics")), - # The separate Comments dataset is more granular than a post read and stays off - # until the user intentionally enables it. - "commentMetrics": bool(cfg.get("commentMetrics")), - "dryRun": bool(cfg.get("dryRun")), - # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so - # every action stored before today keeps saving and keeps running unchanged. - "postGroups": groups, - "maxPosts": max_posts}, None - if kind == "find_records": - table = _s(cfg.get("table"), 60).strip() - if not table: - return None, "the find records action names no database" - cond, cerr = clean_cond(cfg.get("cond"), where="the find") - if cerr: - return None, cerr - try: - limit = int(cfg.get("limit") or 25) - except (TypeError, ValueError): - return None, "the find records limit must be a whole number" - if limit < 1 or limit > FIND_LIMIT_MAX: - return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" - return {"table": table, "cond": cond, "limit": limit}, None - if kind == "ai_agent": - # ⭐⭐ W33-T56 — THE FUZZY STEP'S CONFIG. Same posture as the web arm below in every - # respect that matters: an ALLOWLIST, a shape test on the url, a clamped timeout, and an - # EMPTY config stored rather than refused so the step can be added before it is configured - # (the wave-23 illegal-default-seed defect, and D-79's scar). - # ⚠ IT DELIBERATELY DOES NOT TAKE A `selector`. Working the selector out is the step; a - # selector box here would be a control that contradicts the action's own reason to exist. - _u = _s(cfg.get("url"), 500).strip() - if _u and not (_u.startswith("http://") or _u.startswith("https://") - or _u.startswith("{{")): - return None, "a web address must start with http:// or https://" - try: - _t = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) - except (TypeError, ValueError): - return None, "the web step timeout must be a whole number of milliseconds" - try: - _max = int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS) - except (TypeError, ValueError): - return None, "the step ceiling must be a whole number" - out = {"url": _u, - # ⚠ LONGER THAN A SELECTOR AND SHORTER THAN A PROMPT. This is a sentence or two - # describing a job, and the model is given the page's own vocabulary at run time — - # a 4,000-character instruction is a sign somebody is writing a program in prose. - "instruction": _s(cfg.get("instruction"), 600), - "field": _s(cfg.get("field"), 80).strip(), - "timeoutMs": max(1000, min(_t, WEB_READ_TIMEOUT_MAX_MS)), - # ⛔ THE CEILING IS THE POINT OF THE CLAMP, not the default. Every step this action - # composes is a real browser action inside one job, and a fuzzy instruction is - # exactly the input that produces twenty of them. Bounded here so a run cannot be - # talked into an unbounded journey by a sentence. - "maxSteps": max(1, min(_max, AI_AGENT_MAX_STEPS))} - if cfg.get("dryRun"): - out["dryRun"] = True - return out, None - if kind in WEB_KINDS: - # ⭐⭐ WAVE 31 · T38 (C5 / R10) — AND THIS ARM IS THE HALF THAT WOULD HAVE SHIPPED MISSING. - # - # ⭐⭐ W31 QA WIDENED IT FROM `web_read` TO ALL FIVE KINDS, AND THAT IS THE SAME DEFECT THIS - # ARM'S OWN COMMENT DESCRIBES, ARRIVING A SECOND TIME. When the four write kinds flipped to - # `ready:True` on the owner's revocation of R5, they became storable — and with this arm - # still testing `kind == "web_read"` they fell through to `return {}, None` below, keeping - # their KIND and losing `url` / `selector` / `value` / `hint` **silently, with no error - # anywhere**. Addable, storable, runnable, and forever unconfigured. The paragraph below - # was written about exactly this, one kind earlier; widening the test is what makes it - # true of the feature rather than of one row in it. - # - # ⛔ FOUND BY THE GATE, NOT BY REVIEW, AND IT IS THE TICKET'S OWN FAILURE MODE. Flipping - # `ready:True` and mounting the dispatch arm is not enough: with no branch here the - # function falls through to `return {}, None` below, so a stored `web_read` action keeps - # its KIND and loses its `url`, `selector` and `field` **silently, with no error anywhere**. - # The action would be addable, storable, runnable — and would fetch `""` forever. That is - # precisely the *"addable and inert"* outcome E's hand-off warned about, arriving through a - # different door than the one being watched, and the same silent-drop class as `patch`'s - # hand-maintained key list one screen up. - # - # ⚠ THE URL IS NOT `guard`ed HERE, on purpose. `guard` does DNS, and this is a validator - # that runs on every save of every automation — a save must not block on name resolution, - # and a template like `https://{Domain}/p` is not resolvable until a record supplies the - # value. The rail is enforced where the fetch happens (the job, and `web_agent`'s own - # refusal), which is the only place the FINAL url exists. What is enforced here is the - # shape: http(s) or an interpolation that could become one. - # ⛔⛔ AN EMPTY CONFIG IS STORED, NOT REFUSED — THE A3 STORED-INERT PATTERN, and the first - # draft of this arm got it wrong in a way only a gate could see. Requiring `url` / - # `selector` / `field` here makes the action UNADDABLE: `AutomationBuilder::seedFor` mints - # a new step from a seed and persists it immediately, so a validator that refuses an empty - # seed yields a red banner and no card — the wave-23 illegal-default-seed defect, caught by - # `verify_automation_ui`'s catalog-derived seed check the moment `web_read` became ready. - # ⇒ The same posture the enrich arm already takes with `profileField`: a step may be added - # to a flow before it is configured, and it FAILS CLOSED AT RUN with a sentence naming what - # is missing. What is refused here is a value somebody actually TYPED and got wrong — never - # the absence of one they have not typed yet. - url = _s(cfg.get("url"), 500).strip() - # ⚠ `{{` IS THE TEMPLATE FORM — `interpolate`'s syntax is `{{field_key}}`, NOT `{field}`. - # A url that STARTS with a placeholder is legitimate (the record supplies the host), so the - # shape test admits it; anything else must be an absolute http(s) address. - if url and not (url.startswith("http://") or url.startswith("https://") - or url.startswith("{{")): - return None, "a web address must start with http:// or https://" - selector = _s(cfg.get("selector"), 200).strip() - field = _s(cfg.get("field"), 80).strip() - try: - timeout = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) - except (TypeError, ValueError): - return None, "the web step timeout must be a whole number of milliseconds" - timeout = max(1000, min(timeout, WEB_READ_TIMEOUT_MAX_MS)) - out = {"url": url, "selector": selector, "field": field, - "attr": _s(cfg.get("attr"), 40).strip() or "text", - "all": bool(cfg.get("all")), "timeoutMs": timeout} - wait = _s(cfg.get("waitFor"), 200).strip() - if wait: - out["waitFor"] = wait - # ⭐ THE FOUR WRITE KINDS' OWN KEYS. Carried for EVERY web kind rather than switched on - # `kind`, because the cost of carrying an unused key is nothing and the cost of dropping a - # used one is a step that runs forever against a value the user typed and cannot see. - # `value` is the text `web_fill` types; `secret` masks it in the run log; `hint` is the - # human label `web_repair` follows when a selector has gone stale; `dryRun` rehearses a - # write without performing it (E measured the field reading back empty afterwards). - value = _s(cfg.get("value"), 500) - if value: - out["value"] = value - hint = _s(cfg.get("hint"), 200).strip() - if hint: - out["hint"] = hint - if cfg.get("secret"): - out["secret"] = True - if cfg.get("dryRun"): - out["dryRun"] = True - return out, None - return {}, None - - -def _branch_letter(i): - """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a - label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is - worse than an ugly name.""" - return chr(65 + i % 26) * (1 + i // 26) - - -def group_branches(act): - """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape. - - ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying - for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored - group might carry either shape, and the one that forgot would silently skip a live - automation's actions. A definition cleaned since wave 24 always carries `branches`; one read - straight out of the store (the runner reads definitions, not payloads) may not. - """ - cfg = (act or {}).get("config") or {} - brs = cfg.get("branches") - if isinstance(brs, list) and brs: - return [b for b in brs if isinstance(b, dict)] - return [{"id": "b1", "label": "A", "cond": cfg.get("cond"), - "actions": cfg.get("actions") or []}] - - -def walk_actions(actions): - """Every action in the tree, depth-first, groups included. One walker, so "how many actions - does this flow have" and "which review stages exist" cannot answer differently. - - ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action - COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version - that still read `config.actions` would report a fork's contents as zero actions, hide every - review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing. - """ - for act in actions or []: - yield act - if act.get("kind") == "group": - for br in group_branches(act): - for kid in walk_actions(br.get("actions")): - yield kid - - -def interpolate(text, row): - """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an - action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would - put template syntax in front of a customer, and a blank is the visible failure.""" - if not isinstance(text, str) or "{{" not in text: - return text - def _sub(m): - v = (row or {}).get(m.group(1).strip()) - return "" if v is None else str(v) - return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. -# -# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits -# them once at the end (the 256-commits/hr budget every writer in this module respects). A -# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget -# on one busy flow. -# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the -# runner stamps its stage and stops walking THAT record — later actions belong to the branch a -# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. -# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the -# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write -# cannot fire an event trigger, its own or a sibling's. - -def _act_row_patch(patches, table, rid, values): - patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) - - -def profile_field_key(table, named="", source=PROFILE_SOURCE_IG): - """C3/C4: WHICH COLUMN on this database carries the profile handle. '' when none does. - - Resolution order, and the missing branch is the important one: - 1. the column the action NAMES — the person picked it, so it is not a guess; - 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`; - 3. otherwise **NOTHING**, and the run says so. - - ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url` - column" and is exactly right for `field_instagram`, whose whole configuration was a URL column - — but here it would silently bind the enrich step to whatever URL column happens to be first, - on a database where nobody has said which column is a profile. The failure would be a run that - reports success having enriched from the wrong column, which is worse than one that refuses: - a wrong number is harder to notice than a missing one, and it would be written into the - permanent history as if it had been measured. - - ⭐ WAVE 30 · T08 — `source` PARAMETERISES STEP 2 ONLY, defaulting to Instagram so every - existing caller is byte-for-byte unchanged. Step 1 (the column the person NAMED) is already - network-agnostic: a named column is a decision, not a guess, and second-guessing it against a - flag would refuse a binding somebody made on purpose. - ⛔ AND THE TWO SOURCES MUST NOT BE MERGED INTO "any profile flag". `ut_tt_profile.handle` - carries `profile: {source: "tiktok"}` and `ut_ig_profile.handle` carries `"instagram"`; a - resolver that accepted either would let a TikTok enrich step bind to an Instagram column and - spend money asking a TikTok dataset about an Instagram handle — a wrong number written into a - permanent history, which is the exact failure the no-fallback rule above exists to prevent. - """ - fields = (table or {}).get("fields") or [] - want = str(named or "").strip() - if want: - return want if any(str(f.get("key")) == want for f in fields) else "" - for f in fields: - p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == str(source): - return str(f.get("key") or "") - return "" - - -def _flow_table(defn): - """The database a flow's actions and ending operate on: the automation's own target, or the - table its event trigger watches when the flow has no target of its own. - - ⭐ WAVE 30 · T05 — the discovery arm covers BOTH kinds. It read `== "discover_instagram"`, so a - TikTok discovery with no stored `targetTable` fell to the trigger's table — which a corpus - search does not have — and resolved to `""`. `_presets_after_write` asks this function where to - spawn the preset columns, so an empty answer there is a database that never appears. - """ - cfg = (defn or {}).get("config") or {} - if defn.get("kind") in DISCOVERY_KINDS: - return cfg.get("targetTable") or discovery_facts(defn.get("kind"))[1] - return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") - - -#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN. -#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the -#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]` -#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own. -MAX_ENRICH_PER_RUN = 100 -DEFAULT_ENRICH_LIMIT = 25 -DEFAULT_ENRICH_COOLDOWN_DAYS = 30 -#: Which column the selection orders by when nobody has said. `first_found` newest-first is the -#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what -#: discovery just found rather than re-walking the oldest leads in the book. -DEFAULT_ENRICH_SORT = "first_found" - -#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL. -#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence -#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is -#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored. -#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with -#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by -#: accident, and `_commit_run`'s numeric filter is the second net under it. -RUN_NOTES_KEY = "_notes" -#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES -#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval. -#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence -#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so -#: the verdict should expire. But nothing about US changes on day 31 — the only new information -#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per -#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a -#: hundred stale handles pays a hundred times a month to be told the same thing. -#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors: -#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no -#: wiring at all, because the corrected handle simply is not the one we recorded; and -#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle -#: (a person saying "try it again") also re-arms. -#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered -#: them: a tombstone the owner cannot see is just a disappearance. -#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An -#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the -#: opposite mask ([[gate-answers-the-wrong-question]]). -PENDING_PROFILE_MAX_HOURS = 24 - - -def _order_key(value): - """Order one CELL. Numbers numerically, everything else as text. - - ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text — - which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting - paid for. Numbers are tried first and text is the fallback, never the other way round. - ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where - lexicographic order IS chronological order. If that format ever changes, this comment is the - thing that was relied on. - """ - s = str(value if value is not None else "").strip() - try: - return (0, float(s.replace(",", "")), "") - except ValueError: - return (1, 0.0, s.lower()) - - -def _days_since(day, today=None): - """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read. - - ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a - large number — the cooldown below reads an unreadable stamp as "never enriched", which spends - money rather than skipping. That is the right way round: a skipped record is invisible, and a - stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data. - """ - import datetime as _dt - s = str(day or "").strip()[:10] - if not s: - return None - try: - d = _dt.date.fromisoformat(s) - except ValueError: - return None - return ((today or _dt.date.today()) - d).days - - -def _hours_since(stamp): - """Whole hours since a full ISO stamp, or None when it cannot be read. - - ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date` - column and truncates to 10 characters. A pending snapshot's age is measured in hours and its - stamp carries a time, so reusing that function would read every entry as "queued at - midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid, - outstanding snapshot out of the queue. - """ - import datetime as _dt - s = str(stamp or "").strip() - if not s: - return None - try: - t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00")) - except ValueError: - return None - now = _dt.datetime.now(_dt.timezone.utc) - if t.tzinfo is None: - t = t.replace(tzinfo=_dt.timezone.utc) - return max(0, int((now - t).total_seconds() // 3600)) - - -def _gone_key(row, handle, platform=None): - """The identity a "this account does not exist" verdict is remembered under. - - `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on - Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what - every row on a preset profile table is and what `preset_cells` stamps. - - ⭐ WAVE 30 · T08 — `platform` OVERRIDES THE ROW, and the override is what makes the key honest - on a table the row cannot speak for. The fallback above reads the row's own `platform` cell, - which the TikTok runner writes — but only AFTER the first successful pull. A hand-typed row on - an arbitrary database that a TikTok step is bound to by NAME (`profile_field_key` step 1, - which is deliberately network-blind) carries no such cell, so a TikTok "this account does not - exist" verdict would have been filed under `instagram:`. A tombstone has no expiry - (W28/R9), so that would permanently suppress an Instagram read of a DIFFERENT person's - account with the same handle — silently, and without ever spending the money that would have - shown it was wrong. The caller knows which network it asked; the row does not. - """ - plat = str(platform or (row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() - return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}" - - -def clear_gone(rt, table_key, handle, row=None): - """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle. - - Returns the number of automations whose memory was changed — 0 is the ordinary answer and is - not an error, because most cell edits are not on a dead handle. - - ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER. - The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to, - because it is a fact about what a run learned and paid for. But the thing that re-arms it is a - ROW event, and the row writer must not need to know the shape of an automation's state to - trigger it. So the seam is one call with the three things the writer already has, and every - walk of `all_definitions` stays on this side of the fence. - ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which - handle is this", so a verdict can never be recorded under a key this cannot find - ([[one-evaluator-per-question]]). - - ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first - door — correcting a typo — works with no wiring at all, because the key IS the handle and a - different handle is simply not the one we recorded. This second door only opens when the row - write path calls it, which lives in `routes_tables.patch_row` (another session's fence). - Until that line lands, re-typing the SAME dead handle stays skipped - ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later). - """ - key = _gone_key(row or {}, handle) - if not str(handle or "").strip(): - return 0 - changed = 0 - for auto_id, defn in (all_definitions(rt) or {}).items(): - if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key): - continue - known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {}) - if key not in known: - continue - known.pop(key, None) - # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what - # every other state writer here does and what keeps a definition from growing a graveyard - # of empty maps. - set_state(rt, str(auto_id), {"enrichNotFound": known or None}) - changed += 1 - return changed - - -def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch, - step=_no_step, log=print): - """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved. - - ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a - LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an - hour"); the only profile caller passed a single-element list from inside the per-record walk. - MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25 - vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand. - - ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from - `prefetch`, and the per-record rung then behaves exactly as it does today — including its - corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a - batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure - returns 0 instead of raising. - - ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now - serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they - share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the - handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch` - behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time. - """ - want, dests, handles = [], {}, [] - for rid in (chosen or []): - row = rows.get(str(rid)) or {} - h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower() - if not h or h in prefetch: - continue - want.append((h, str(rid))) - if h not in dests: - handles.append(h) - dests.setdefault(h, []).append(str(rid)) - if not handles: - return 0 - step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source") - deferrals = [] - try: - nodes, note = bd_profiles_batch(handles, deferred=deferrals) - except Exception as exc: # noqa: BLE001 — never fatal, see docstring - log(f"[aios-auto] profile batch failed, falling back to per-record reads: " - f"{type(exc).__name__}: {exc}") - return 0 - prefetch.update(nodes) - if note: - log(f"[aios-auto] profile batch note: {_s(note, 300)}") - for d in deferrals: - sid = str(d.get("snapshotId") or "") - if not sid: - continue - for u in (d.get("urls") or []): - h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower() - if not h or h in nodes: - continue - prefetch[h] = {DEFERRED_MARK: sid} - for rid in dests.get(h, []): - enrich["pendingProfiles"].append( - {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""), - "urls": [str(u)], "kind": "profile", "influencer": h, - "table": table, "rowId": rid, "requestedAt": _iso()}) - return len(nodes) - - -def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, platform=None): - """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order. - - Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did - and is surfaced in the run summary; `""` means there is nothing worth saying. - - ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's - ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is - enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every - skipped record until 30 have actually been enriched or the list runs out. The naive reading — - take the top 30, then filter — would have billed for 10 and reported success, and the number in - the box would have meant something different every run depending on how much of the top of the - list happened to be fresh. A quota is predictable; a filtered window is not. - - The order of operations, each step narrowing the one above: - 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view` - and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an - empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich - the entire database", at vendor prices. - 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions - (a blank is unknown, not smallest). - 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on. - 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`. - - ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is - nothing to enrich and it must not consume a slot somebody paid for. - """ - t = ut_get(rt, str(table_key or "")) - if t is None: - return [], f"{table_key!r} is not a database in this workspace" - rows = dict(t.get("rows") or {}) - notes = [] - - view_id = str(cfg.get("fromView") or "").strip() - if view_id: - tree, fields, problem = view_filter(rt, table_key, view_id) - if problem: - # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet - # fallback costs real money. - return [], f"{ENRICH_VIEW_UNREADABLE}. {problem}" - # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS - # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE. - # - # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}], - # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/ - # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition — - # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on - # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`, - # got `""`, and answered **False for every row**: MEASURED on the owner's own automation, - # a view matching exactly one record selected NOTHING, reported NO error, and the run - # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual - # record... how come it says 51 records walked?"* - # - # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch - # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has - # always been correct. `_seed_event_state`'s own docstring states the law this line broke: - # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it - # is supposed to arm."* There were three. Now there are two callers of one function. - # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date - # as a date and a number as a number, which is the half `lane_match` could not have had. - import harness.filter_eval as filter_eval - rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)} - - sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT - newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc" - # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN". - # - # Owner: *"treat a missing first found as now, and manual entry should go first, instead of - # treated as last. I want to see my inayma manual entry works with automation enrichment."* - # And they are right about the semantics, not just the preference: on these tables the ONLY - # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it - # on every row it writes. So a blank is not missing data — it is a row that was first found - # today, by the person sitting in front of it, and the one they most want enriched. - # - # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position - # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one - # it never reached. - # - # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about - # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and - # treating it as the maximum would spend the budget on the rows we know least about. So a - # non-date sort keeps the old rule: unknown sorts last, in both directions. - # - # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real - # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at - # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One - # rule, both directions, no branch that can disagree with itself. - ftype = str(((next((f for f in (t.get("fields") or []) - if str(f.get("key")) == sort_field), None)) or {}).get("type") or "") - if ftype == "date": - ordered = sorted(rows.items(), - key=lambda x: (0, str(x[1].get(sort_field) or "").strip()) - if str(x[1].get(sort_field) or "").strip() else (1, ""), - reverse=newest_first) - else: - have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()] - blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()] - have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first) - blank.sort(key=lambda x: _rid_num(x[0])) - ordered = have + blank - - try: - quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) - except (TypeError, ValueError): - quota = DEFAULT_ENRICH_LIMIT - quota = max(1, min(quota, MAX_ENRICH_PER_RUN)) - if cfg.get("limit") and quota != int(cfg.get("limit") or 0): - notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run") - - cooling = bool(cfg.get("skipRecent")) - try: - days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) - except (TypeError, ValueError): - days = DEFAULT_ENRICH_COOLDOWN_DAYS - days = max(1, days) - - picked, cooled, blank_handle = [], 0, 0 - # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST. - # - # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT. - # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is - # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it - # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at - # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix - # removes that loop; only a memory of the verdict does. - # - # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the - # note goes into the run every single time, not only on the run that discovered it, because a - # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf". - skipped_gone = [] - for rid, r in ordered: - if len(picked) >= quota: - break - raw_handle = str(r.get(profile_key) or "").strip() - if not raw_handle: - blank_handle += 1 - continue - # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is - # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the - # owner reads to know when we last paid to be told this), it is simply not a clock. - # ⚠ WAVE 30 · T08 — the SAME `platform` the runner will file a new verdict under, so the - # skip and the write cannot disagree about which account a tombstone belongs to. - if isinstance((gone or {}).get(_gone_key(r, raw_handle, platform)), dict): - skipped_gone.append(raw_handle) - continue - if cooling: - since = _days_since(r.get("enriched_at"), today=today) - if since is not None and since < days: - cooled += 1 - continue - picked.append(str(rid)) - - # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the - # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and - # `run_plain` follow. Silence here would make a shrinking selection invisible. - if cooled: - notes.append(f"{cooled} skipped as enriched in the last {days} days") - if skipped_gone: - shown = ", ".join(f"@{h}" for h in skipped_gone[:5]) - # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can - # do, because nothing else will ever retry them — under R9 this note is the only path - # back from a tombstone, so a vaguer version would strand the row permanently. - notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account " - f"({shown}{', …' if len(skipped_gone) > 5 else ''}). Delete the row or " - f"correct the handle; they are not retried automatically") - if blank_handle: - notes.append(f"{blank_handle} skipped with no handle") - if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered): - notes.append(f"{len(picked)} of the {quota} asked for. The list ran out") - return picked, "; ".join(notes) - - -def _has_action(actions, kind): - """Does this flow contain `kind` ANYWHERE, forks included? - - ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under - `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an - enrich step somebody put inside an If — and the caller would then skip a schema top-up the - run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24 - recorded when four hand-rolled walks all forgot to descend. - """ - for a in (actions or []): - if not isinstance(a, dict): - continue - if a.get("kind") == kind: - return True - for br in ((a.get("config") or {}).get("branches") or []): - if _has_action((br or {}).get("actions") or [], kind): - return True - return False - - -def _actions_of_kind(actions, kind): - """Every action of `kind`, including actions nested inside If branches.""" - out = [] - for action in actions or []: - if not isinstance(action, dict): - continue - if action.get("kind") == kind: - out.append(action) - for branch in ((action.get("config") or {}).get("branches") or []): - out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind)) - return out - - -def _web_agent(): - """C5's seam, resolved LAZILY — `web_agent.run_step(step, ctx) -> (dict|None, str)`. - - ⚠ IMPORTED INSIDE THE CALL, like every `connectors_tt` site in this module, and here it also - buys a failure mode worth having: if `web_agent.py` is ever missing from a deployment, the - engine still imports and every OTHER action still runs — the web step alone reports a sentence. - A module-level import would turn one absent file into a dead automation module. - - ⛔ AND THE ABSENCE IS REPORTED, NEVER SWALLOWED (R6's second sentence). The shim below answers - the same `(None, sentence)` contract the real seam does, so the caller's `if why:` branch is - the only branch there has ever been. - """ - try: - import web_agent - return web_agent - except Exception as exc: # noqa: BLE001 - class _Absent: - @staticmethod - def run_step(_step, _ctx=None): - return None, ("The web-browsing agent is not available in this deployment " - f"({type(exc).__name__}). Nothing was read.") - return _Absent - - -def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print, - step=_no_step): - """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. - - ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery - automation's review stage comes from `stages_for`, not from actions, - so an early return on an empty action list would have silently disabled the owner's loop - feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in - review; the tests all happened to pass a non-empty action list, which is why it was green. - """ - defn, _retired = _without_retired_board(defn) - flow = (defn or {}).get("flow") or {} - actions = flow.get("actions") or [] - # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE. - # - # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the - # "Save results" node) and THEN walked the flow over the rows it had just written — where - # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on - # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner - # ruling, same day) would have made the duplication permanent and unavoidable with it. - # - # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what - # it finds in a database" is exactly what the engine does. What it must not do is do it - # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the - # server still refuses to let it be removed. - if ig_action_pinned(defn, 0) and actions: - actions = actions[1:] - # ⭐ `webRead` / `webBlocked` JOIN HERE (W31-T38, C5): declared up front like every other - # counter, so a run that read nothing and a run with no web step are DIFFERENT numbers. R6's - # second sentence is a reporting rule, and a counter that only exists once it is non-zero - # cannot report a refusal. - counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0, - "webRead": 0, "webBlocked": 0} - #: W31-T38 — the once-per-run markers the web arm uses, so an unconfigured step says so ONCE - #: rather than once per record. Same shape as the enrich accumulators' `notes`. - web_notes = [] - # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected - # across the WHOLE walk and written once at the end, for the reason the module header states - # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on - # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an - # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts. - enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], - # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor) - # and handles a vendor has said do not exist. Both are collected across the whole - # walk and written once, for the same reason every other list here is. - "pendingProfiles": [], "gone": {}, - "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} - # ⭐⭐ WAVE 30 · T08 — THE TIKTOK ACCUMULATOR IS ITS OWN, and the separation is the design - # decision rather than a convenience. `_enrich_flush` is not network-agnostic: it calls - # `ensure_ig_graph`, addresses the four `IG_*_TABLE` constants and writes through to the - # platform Instagram master (`ig_master.append_run`). Routing TikTok rows through it would - # append a TikTok creator's followers into Instagram's pooled history, which the metric - # FIELDS read (W29-T34) — a wrong number in a permanent series, i.e. the same failure class - # as `preset_cells`' unconditional `PLATFORM_INSTAGRAM` stamp that `tt_preset_cells` exists - # to avoid. - # ⚠ AND THE KEYS IT FLUSHES ARE PREFIXED `tt…` FOR A MEASURED REASON: `apply_actions` merges - # both flushes into ONE `counts` dict, so a shared `enriched`/`enrichBlocked` key would let a - # flow carrying BOTH an Instagram and a TikTok step report one network's numbers under the - # other's name, last writer winning, with no error anywhere. - # ⭐ W30 · D-156 — `pending` is the TikTok twin of Instagram's, and it is a RUN-level list for - # the same reason theirs is: a media snapshot belongs to a HANDLE, not to a row, so nothing - # about it needs the per-record sink that `pendingProfiles` needs. - tt = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], - "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} - # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them - # per record and re-reading the definition per row would be a store read per record. - known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) - # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD. - # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk - # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked - # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps - # in one flow are two independent budgets. - enrich_plan = {} - # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK. - # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a - # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was - # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever - # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are - # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth - # buying twice. - enrich_prefetch = {} - if not table_key: - return counts - table = str(table_key) - # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN. - # - # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled - # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a - # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door. - # So every preset the target table did not already happen to have was silently dropped, and - # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET - # FILLED"* — this is the half that was swallowing them. - # - # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`, - # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose - # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the - # feature worked on discovery tables purely because discovery creates them for its own reasons. - # - # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present: - # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady - # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no - # enrich step never touches the schema of the database it walks. - # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset - # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an - # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a - # step at it, and (b) add a SECOND profile column to a table that already flagged a different - # one, which `user_tables` forbids at both write doors. A table with no profile column is an - # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its - # own honest refusal, not something to paper over by inventing the binding. - _tbl_now = ut_get(rt, table) or {} - _enrich_actions = _actions_of_kind(actions, "enrich_instagram") - _bound = next((f for f in (_tbl_now.get("fields") or []) - if isinstance(f.get("profile"), dict)), None) - if _bound is None: - _named = next((str((a.get("config") or {}).get("profileField") or "").strip() - for a in _enrich_actions - if str((a.get("config") or {}).get("profileField") or "").strip()), "") - _bound = next((f for f in (_tbl_now.get("fields") or []) - if str(f.get("key") or "") == _named), None) - if _bound and _enrich_actions: - # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists — - # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a - # table whose profile column is `ig_handle` would otherwise gain a rival `handle`. - topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")} - if f.get("key") != _bound.get("key") else dict(f)) - for f in PRESET_PROFILE_FIELDS] - try: - if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions): - ensure_ig_graph(rt, username, str(defn.get("id") or ""), - profile_table=table, profile_field=str(_bound.get("key") or "")) - else: - # Dry run keeps its original no-create contract; only the already-established - # Profile table is topped up, and no canonical related database is spawned. - ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table, - flow_tag=str(defn.get("id") or ""), lock_fields=True) - except Exception as exc: # noqa: BLE001 - # A schema top-up that cannot run must not stop the enrichment: the cells that DO - # have columns still land, which is strictly better than the pull being thrown away. - log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}") - # ⭐⭐ WAVE 30 · T08 — THE SAME TOP-UP FOR TIKTOK, AND THE SAME NARROWING: only on a table that - # is ALREADY BOUND. Separate from the block above because every constant in it is Instagram's - # (`PRESET_PROFILE_FIELDS`, `ensure_ig_graph`) and because the binding is resolved by a - # DIFFERENT question — `profile_field_key(..., source=PROFILE_SOURCE_TT)`, so a TikTok step - # cannot adopt an Instagram column and go on to ask a TikTok dataset about an Instagram handle. - # ⚠ It runs on a dry run too, exactly as the Instagram branch does: the target's own columns - # are topped up, and the related append database is NOT spawned (that is the flush's job, and - # it returns before writing anything on a dry run). - _tt_actions = _actions_of_kind(actions, "enrich_tiktok") - if _tt_actions: - _tt_named = next((str((a.get("config") or {}).get("profileField") or "").strip() - for a in _tt_actions - if str((a.get("config") or {}).get("profileField") or "").strip()), "") - _tt_bound = profile_field_key(_tbl_now, _tt_named, source=PROFILE_SOURCE_TT) - if _tt_bound: - try: - ut_ensure(rt, _tbl_now.get("label") or table, - _tt_profile_schema_for(_tt_bound), username, key=table, - flow_tag=str(defn.get("id") or ""), lock_fields=True) - except Exception as exc: # noqa: BLE001 - log(f"[aios-auto] TikTok preset top-up on {table} failed: " - f"{type(exc).__name__}: {exc}") - rows = dict((ut_get(rt, table) or {}).get("rows") or {}) - if not rows: - return counts - # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The - # walk used to keep a `reached` map and stamp each record with the furthest action it made — - # a board column's worth of bookkeeping written into the tenant's own table on every run. - # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a - # cell the customer has to look at and a store commit we pay for on every scheduled run. - # The RUN LOG is the operational record of what happened now, and it is the only one. - patches, creates = {}, {} - - #: ⭐⭐ W33-T58 · D-191 — this record's web results, keyed by ACTION ID, and the run's job - #: count. `web_done` is cleared per record (a batch is built from one row's interpolated - #: values and means nothing for the next one); `web_jobs` counts JOBS across the whole run, - #: which is what `MAX_WEB_JOBS_PER_RUN` has always been trying to bound. - #: ⚠ A key present with `None` means "this step was in a batch that refused" — distinct from - #: absent, which means "no batch has covered it yet". Collapsing the two would re-run the - #: whole batch once per step of it, which is the defect this fixes, inverted. - web_done: dict = {} - web_jobs = [0] - - def _web_step_dict(act, row): - """One action + one record → the step dict the seam takes. Interpolated HERE, as before. - - ⚠ `interpolate` on the CALLER's side, exactly as the update/create arms do it, so - `{{Field}}` works in a url, a selector or a typed value. The seam takes a plain dict and - does not know about rows. - """ - cfg = act.get("config") or {} - wait = cfg.get("waitFor") - step = {"kind": act.get("kind"), "id": str(act.get("id") or ""), - "url": interpolate(str(cfg.get("url") or ""), row), - "selector": interpolate(str(cfg.get("selector") or ""), row), - "attr": str(cfg.get("attr") or "text"), - "all": bool(cfg.get("all")), - "waitFor": interpolate(str(wait), row) if wait else None, - "timeoutMs": int(cfg.get("timeoutMs") or 20000)} - # ⚠ ADDED ONLY WHEN PRESENT: the seam distinguishes a key that is absent from one that is - # empty, and an empty `value` on a `web_read` would be a typed blank. - if cfg.get("value"): - step["value"] = interpolate(str(cfg.get("value")), row) - if cfg.get("hint"): - step["hint"] = interpolate(str(cfg.get("hint")), row) - if cfg.get("secret"): - step["secret"] = True - if cfg.get("dryRun"): - step["dryRun"] = True - return step - - def _run_web_batch(acts, start, row, rid): - """Run the longest safe run of consecutive web steps from `acts[start]` in ONE job. - - ⛔ **CONSECUTIVE, AND ONLY WHILE NOTHING IN THE BATCH DEPENDS ON THE BATCH.** `run_plan` - sends every step to one browser at once, so a step whose url/selector/value interpolates a - column an EARLIER step in the same batch writes would be interpolated against the value - that column had BEFORE the batch ran. That is a wrong answer rather than a slow one, so the - batch is cut immediately before any such step and the remainder becomes the next batch. - The single-step case is then exactly the old behaviour, which is what makes this safe to - land on a live flow. - ⚠ A step that is disabled, filtered out by its `when`, or unconfigured ENDS the batch - rather than being skipped inside it: each of those is a reason this record does not run - that step, and the loop's own arms already report them one at a time with their own - sentences. Ending here keeps exactly one place that decides what a blocked step says. - """ - batch, produced = [], set() - for act in acts[start:]: - kind = act.get("kind") - if kind not in WEB_KINDS or not act.get("enabled", True): - break - if not lane_match(act.get("when"), row): - break - cfg = act.get("config") or {} - if _web_missing(kind, cfg): - break - # The dependency cut. `interpolate` reads `{{Name}}`; a step naming a column an - # earlier step in THIS batch writes has to wait for the next job. - refs = " ".join(str(cfg.get(k) or "") for k in ("url", "selector", "value", "hint")) - if any(("{{" + f) in refs or ("{{ " + f) in refs for f in produced): - break - batch.append(act) - if str(cfg.get("field") or ""): - produced.add(str(cfg.get("field"))) - if len(batch) >= MAX_WEB_STEPS_PER_JOB: - break - if not batch: - return - steps = [_web_step_dict(a, row) for a in batch] - # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to - # act on yet — and a refusal is for the WHOLE plan. Batching a url-less first step would - # therefore take its followers down with it, where one-job-per-step only lost that step. - # A batch that cannot start is cut to one, which is exactly the old behaviour. - if not str(steps[0].get("url") or "").strip(): - batch, steps = batch[:1], steps[:1] - web_jobs[0] += 1 - - def _block(a, why_one): - web_done[str(a.get("id") or "")] = None - counts["webBlocked"] += 1 - if why_one: - log(f"[aios-auto] {a.get('kind')}: {why_one}") - - # ⛔ THE SEAM'S OWN BOUNDARY IS ON `run_step`, NOT ON `run_plan` — `run_step` wraps its - # call in `try/except` and calls that "the LAST boundary". Calling `run_plan` directly - # steps around it, and this code runs inside a record walk where an escaping exception - # ends the whole run. So the boundary moves here with the call. - try: - rows_out, why = _web_agent().run_plan( - steps, {"tenant": str(defn.get("tenant") or ""), - "automationId": str(defn.get("id") or ""), - "runId": str(defn.get("id") or ""), "log": log}) - except Exception as exc: # noqa: BLE001 — the LAST boundary - why, rows_out = (f"The web steps failed unexpectedly ({type(exc).__name__}: " - f"{str(exc).splitlines()[0][:200]}). Nothing was read."), None - if why: - log(f"[aios-auto] web: {why}") - for a in batch: - _block(a, "") - return - # ⚠ MATCHED BY ID, NEVER BY POSITION. The job returns a row for a step that FAILED and for - # every step after it that was never attempted, so the list can be shorter than, or - # misaligned with, the plan — and a positional read would hand step 3's caller step 2's - # answer, writing a wrong value into a real column, which is worse than the missing one it - # replaced. `_clean_step` mints an id for every step and the runner echoes it back, so the - # id is carried the whole way and is the only thing worth matching on. - by_id = {str(r.get("id") or ""): r for r in (rows_out or []) if isinstance(r, dict)} - for a in batch: - hit = by_id.get(str(a.get("id") or "")) - # ⛔ `ok` IS CHECKED HERE BECAUSE `run_step` USED TO CHECK IT. It turned a row with - # `ok:false` into a sentence and returned no result; reading `hit` without that test - # would take a failed step's empty `value` and write it over a real cell. - if not hit or not hit.get("ok"): - _block(a, str((hit or {}).get("error") or "") - or "The browser job returned nothing for this step.") - continue - web_done[str(a.get("id") or "")] = hit - - def _walk(acts, row, rid, depth=0): - """Walk the actions in order for ONE record. - - ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean - "this record was SUSPENDED by a review gate" — the one branch that could stop a walk - early. With review retired nothing suspends anything, so every record walks its whole - flow; the signature stays so a future gate-style action has an obvious place to say so. - """ - for idx, act in enumerate(acts): - if not act.get("enabled", True): - continue - if not lane_match(act.get("when"), row): - continue - kind = act.get("kind") - cfg = act.get("config") or {} - counts["actionsRun"] += 1 - if kind == "group": - # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs. - # Declaration order is priority order, the same rule `route_record` applies to - # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch - # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions` - # has already guaranteed a null condition can only be LAST. - for br in group_branches(act): - if lane_match(br.get("cond"), row): - if _walk(br.get("actions") or [], row, rid, depth + 1): - return True - break - elif kind == "update_record": - vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} - row.update(vals) # later actions see the write, as they must - _act_row_patch(patches, table, rid, vals) - counts["updated"] += 1 - elif kind == "create_record": - target = str(cfg.get("table") or "") - vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} - # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions - # may legitimately write to ONE database on different keys — collapsing them onto - # the table would silently apply one action's uniqueness rule to the other's rows. - creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) - counts["created"] += 1 - elif kind == "send_statement": - # ⭐⭐ WAVE 35 · T35 / R10 — THE ARM LANDS WITH THE CATALOG ROW, ON PURPOSE. - # - # `_walk` has no terminal `else` (see the note on `enrich_tiktok` in the catalog): - # an unknown kind is walked, COUNTED, reports the run `ok` and writes nothing. So a - # catalog row whose arm arrives in a later ticket is addable, storable and silently - # inert — a step a person configured, that reports success and does nothing. This - # arm exists so that window never opens. - # - # ⛔ T35 DOES NOT SEND AND DOES NOT PARK. R10's review stage is W35-T36; until it - # lands this says so out loud and counts the record as blocked, which is the same - # shape `ai_agent` uses for a step it cannot perform. It must never fall through to - # "ok". - # ⛔ AND IT NEVER SENDS FROM HERE, in this wave or any later one. The send door is - # `routes_statements`, behind SAFE_MODE + `admin_gate` + the tenant gate, reached by - # a human click on the review batch. This arm's whole job is to PREPARE. - if "send_statement_pending" not in web_notes: - web_notes.append("send_statement_pending") - log("[aios-auto] send_statement: statements are assembled for review, not " - "sent. The review batch is not configured yet, so nothing was prepared " - "and nothing was sent.") - counts["webBlocked"] += 1 - continue - elif kind == "odoo_sync": - # Same refusal, same reason as the arm below: a catalog row with no arm is walked, - # counted, and reports success having done nothing. The connector owns this sync. - if "odoo_sync_not_run_here" not in web_notes: - web_notes.append("odoo_sync_not_run_here") - log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own " - "schedule, not from this canvas. Nothing was done") - counts["webBlocked"] += 1 - continue - elif kind == "ai_enrich": - # ⛔⛔ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE - # STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is - # walked, COUNTED, reports the run `ok` and writes nothing — a step somebody - # configured that succeeds at doing nothing. Adding the row (D-277) without this - # arm would have opened exactly that window. - # ⚠ AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by - # `ai_enrich`, driven from the column's own editor; the synthetic agent row this - # kind appears in is DERIVED from that column and is never stored, so nothing - # reaches here through the ordinary path. If something ever does, it must say so - # rather than report success. - if "ai_enrich_not_run_here" not in web_notes: - web_notes.append("ai_enrich_not_run_here") - log("[aios-auto] ai_enrich: an AI column is filled from the column's own " - "editor, not from this canvas. Nothing was done") - counts["webBlocked"] += 1 - continue - elif kind == "ai_agent": - # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. - # - # A description becomes concrete web steps HERE, against this record's own values, - # and then rides the ordinary seam. Composing at run time rather than at save time - # is the whole point: `{{Website}}` is a different page for every row, so a journey - # fixed at save time would be the same guess repeated. - # ⛔ IT COMPOSES ONLY `web_*` KINDS. The composer is handed the same catalog the - # AI-agent module uses, filtered to what a browser job can perform — so a fuzzy - # instruction cannot talk this action into writing a record or calling a connector. - # The blast radius of a bad sentence is one browser session, not the tenant. - # ⚠ AND IT REPORTS THE STEPS IT ACTUALLY TOOK, which is the ticket's own - # `done-when`. A step that composes a journey and reports only its final value is - # unauditable: nobody can tell a right answer from a lucky one. - _missing = _web_missing(kind, cfg) - if _missing: - _note = f"{kind}_unconfigured" - if _note not in web_notes: - web_notes.append(_note) - log(f"[aios-auto] {kind}: this step still needs " - + ", ".join(_missing) + ". Nothing was done") - counts["webBlocked"] += 1 - continue - if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: - if "web_cap" not in web_notes: - web_notes.append("web_cap") - log(f"[aios-auto] web: this run stopped after {MAX_WEB_JOBS_PER_RUN} " - f"browser jobs of about 10-30 seconds each.") - counts["webBlocked"] += 1 - continue - # W35 · C7: `st` + `user` so the model spend is attributed (`NOTE E-16`). - _plan, _why = _ai_agent_plan(cfg, row, log, st=rt, - user=str(defn.get("createdBy") or "")) - if _why: - # ⛔ NAMED, NEVER OPAQUE — the second half of the `done-when`. "The assistant - # could not work out how to do that" with the reason attached is actionable; - # a blank cell is not. - log(f"[aios-auto] ai_agent: {_why}") - counts["webBlocked"] += 1 - continue - web_jobs[0] += 1 - try: - _rows_out, _why2 = _web_agent().run_plan( - _plan, {"tenant": str(defn.get("tenant") or ""), - "automationId": str(defn.get("id") or ""), - "runId": str(defn.get("id") or ""), "log": log}) - except Exception as _exc: # noqa: BLE001 — the LAST boundary - _rows_out, _why2 = None, ( - f"the browser job failed unexpectedly ({type(_exc).__name__}: " - f"{str(_exc).splitlines()[0][:200]}). Nothing was done.") - if _why2: - log(f"[aios-auto] ai_agent: {_why2}") - counts["webBlocked"] += 1 - continue - # THE ACCOUNT OF WHAT IT DID — one line per step, in order, with each step's own - # verdict. This is what makes a fuzzy step auditable at all. - _done = [r for r in (_rows_out or []) if isinstance(r, dict)] - for _i, _r in enumerate(_done, 1): - log(f"[aios-auto] ai_agent step {_i}/{len(_plan)}: {_r.get('kind')} " - f"{'ok' if _r.get('ok') else 'FAILED'}" - + (f". {str(_r.get('error'))[:160]}" if not _r.get("ok") else "")) - _last = _done[-1] if _done else {} - if not _done or not _last.get("ok"): - log("[aios-auto] ai_agent: the journey did not finish. " - + str((_last or {}).get("error") - or "the browser job returned nothing for the last step")) - counts["webBlocked"] += 1 - continue # ⛔ NOTHING IS WRITTEN on an unfinished journey. - _target = str(cfg.get("field") or "") - if _target: - vals = {_target: _last.get("value")} - row.update(vals) - _act_row_patch(patches, table, rid, vals) - counts["webRead"] += len(_done) - elif kind in WEB_KINDS: - # ⭐⭐ WAVE 31 · T38 (C5) — THE WEB-BROWSING AGENT'S LIVE ARM, ALL FIVE KINDS. - # - # ⛔ WHY THIS ARM EXISTS SEPARATELY FROM THE RUNNER: session E built - # `web_agent.run_step` and **could not verify its own mounting**. An unmounted - # runner is a whole, correct, unreachable feature — the exact class five wave-29 - # features shipped as — so the mount and its `verify_wiring` row are C's, in one - # change. - # ⭐⭐ W31 QA WIDENED THIS ARM FROM `web_read` TO ALL FIVE KINDS on the owner's - # revocation of D-51/R5 (`TICKETS.md:1418`). ⚠ IT NEEDS NO PER-KIND HANDLING, and - # that is E's design rather than an omission: the runner normalises EVERY kind to - # set `result["value"]` (read → the text · goto → the title · click → the title it - # landed on · fill → the typed value, masked when secret · repair → the proposed - # selector), precisely so this one arm does not grow a switch that would be a - # second copy of the runner's table living in another lane's file. - # - # ⛔ IT BLOCKS FOR ~9-32 s (E measured it; `proto/web-agent-job.md` §4). That is - # tolerable HERE and nowhere else: this is the automation RUNNER, already a - # background walk. It must never be called from a route a person is waiting on. - # - # ⚠ `interpolate` ON THE CALLER'S SIDE, exactly as the update/create arms do it, so - # `{Field}` works in a url or a selector. E's seam takes a plain dict and does not - # know about rows. - # ⛔ FAIL CLOSED, ONCE, WITH THE REASON — the counterpart to the validator storing - # an unconfigured step (see `_clean_action_config`'s `web_read` arm). Reported once - # per RUN and not per record: the missing config is a property of the flow, so a - # 100-record walk would otherwise print the same sentence a hundred times and bury - # everything else. The same shape the enrich arm's `unbound` note uses. - # ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and - # no selector; `web_fill` needs a value nobody else takes; only `web_read` needs a - # column to write into. One shared three-field test would have blocked every - # `web_goto` ever configured for want of a selector it does not use. - _missing = _web_missing(kind, cfg) - if _missing: - # ⚠ THE NOTE KEY CARRIES THE KIND. It used to be the literal - # `"web_read_unconfigured"`, so a flow with an unconfigured `web_goto` AND an - # unconfigured `web_fill` would have reported the first and swallowed the - # second — once-per-RUN is the property, not once-per-FLOW. - _note = f"{kind}_unconfigured" - if _note not in web_notes: - web_notes.append(_note) - log(f"[aios-auto] {kind}: this step still needs " - + ", ".join(_missing) + ". Nothing was done") - counts["webBlocked"] += 1 - continue - # ⛔⛔ THE PER-RUN JOB CEILING (E-4). A browser job is ~9-32 s against HF's - # 6-concurrent cap — a flow over a few thousand rows would submit a few thousand - # jobs and run for days. Reported ONCE with its cause AND the fix, which is R6's - # second sentence: a limit that cannot be removed today must say why and what would - # remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning. - # ⭐⭐ W33-T58 (D-191) — THE CEILING NOW COUNTS **JOBS**, NOT PAGES, because the - # two stopped being the same thing on the line below. A record whose three web - # steps batch into one job spends ONE of these, not three. - if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: - if "web_cap" not in web_notes: - web_notes.append("web_cap") - log(f"[aios-auto] web_read: this run stopped after " - f"{MAX_WEB_JOBS_PER_RUN} browser jobs of about 10-30 seconds each, " - f"which do not run in parallel. A record's consecutive web steps " - f"already share ONE job; to read more, narrow the flow's records.") - counts["webBlocked"] += 1 - continue - # ⭐⭐ W33-T58 · D-191 — ONE JOB FOR A RECORD'S CONSECUTIVE WEB STEPS. - # `run_plan(steps, ctx)` has always taken a list and nothing ever called it with - # more than one: the arm called `run_step`, which wraps `[step]`, so a flow with - # three web reads paid THREE ~9 s cold starts to do what one job does. The batch is - # built at WALK time rather than from the stored flow, because whether a step runs - # at all depends on this record (`enabled`, `when`, and whether it is configured). - if str(act.get("id") or "") not in web_done: - _run_web_batch(acts, idx, row, rid) - result = web_done.get(str(act.get("id") or "")) - if result is None: - continue # its batch refused; the reason was logged once, there - _target = str(cfg.get("field") or "") - if _target: - vals = {_target: (result or {}).get("value")} - row.update(vals) # later actions see the write, as they must - _act_row_patch(patches, table, rid, vals) - counts["webRead"] += 1 - elif kind == "enrich_instagram": - # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions - # `run_field_instagram` calls, not a second implementation of either. - pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField")) - if not pkey: - # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a - # property of the flow, so a 100-record run would otherwise put the same - # sentence in the log a hundred times and bury everything else. - if "unbound" not in enrich["notes"]: - enrich["notes"].append("unbound") - log("[aios-auto] enrich_instagram: no profile column on " - f"{table}. Name one on the action, or mark a text column as an " - "Instagram profile") - continue - # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET? - # Resolved once (see `enrich_plan`) and then a membership test. A record outside - # the selection is NOT an error and NOT a skip worth logging per row — it is simply - # not this run's work, and the selection's own note already accounts for it. - aid = str(act.get("id") or "") - if aid not in enrich_plan: - chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone) - enrich_plan[aid] = set(chosen) - if sel_note: - enrich["notes"].append(sel_note) - # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else: - # this is the only place the full chosen set and the row bodies are both in - # hand. Anything it resolves the per-record rung below reads from memory. - _prime_enrich_batch(chosen, rows, pkey, table, enrich, - enrich_prefetch, step, log) - if str(rid) not in enrich_plan[aid]: - continue - handle_raw = str(row.get(pkey, "") or "").strip() - if not handle_raw: - continue # nothing to enrich on this record, not an error - # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED. - # A handle already in `enrich_prefetch` was bought by the batch above and is read - # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the - # batching win straight back (25 records = ~62 s of pure waiting). - if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \ - not in enrich_prefetch: - time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does - enrich["profiles"] += 1 - # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with - # the row it belongs to: the collector writes preset cells back onto THAT record, - # and a run-wide list would have no way to say which handle each snapshot was for. - pend_prof = [] - # ⭐ W31-T39(c) — the DEFERRAL WATERMARK, taken before the pull. See the `partial` - # report below: a capability that was deferred is not a capability that failed, - # and `pull_profile` appends into these two sinks from inside the call. - _pend_before = len(enrich["pending"]) - res = pull_profile(handle_raw, - max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, - log=log, - post_metrics=bool(cfg.get("postMetrics")), - comment_metrics=bool(cfg.get("commentMetrics")), - pending_metrics=(enrich["pending"] - if not cfg.get("dryRun") else None), - pending_profile=(pend_prof - if not cfg.get("dryRun") else None), - prefetch=enrich_prefetch, - post_groups=cfg.get("postGroups")) - # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels / - # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is - # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped. - # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this - # costs nothing and changes nothing until somebody configures it. - # ⭐⭐ WAVE 30 · T16 — THE WINDOW IS NOW WIDENED BEFORE IT IS FILTERED, and the - # paragraph that used to sit here explaining why it could not be is gone with the - # reason. It said: *"it FILTERS the window we already bought … the views top-up - # runs INSIDE `pull_profile` against the posts it captured, so swapping a wider - # list in afterwards would hand back posts with no view counts."* Correct, and - # exactly why the widening went INTO `pull_profile` (`post_groups=`, above) rather - # than being bolted on out here — ahead of the top-up, not after it. - # ⚠ THIS LINE STAYS, and it is not redundant. A single-type ask is served by a - # native route that returns only that type; a MIXED ask has no such route, so the - # wide window comes back mixed and this is what keeps N of each. - if cfg.get("postGroups"): - res = {**res, "posts": select_post_groups(res.get("posts") or [], - cfg.get("postGroups"))} - for _p in pend_prof: - _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() - enrich["pendingProfiles"].extend(pend_prof) - pulled = _iso() - if res["state"] in ("ok", "partial"): - enrich["ok"] += 1 - snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) - enrich["snaps"].append(snap_row) - enrich["posts"].extend(idents) - enrich["psnaps"].extend(metric_rows) - enrich["comments"].extend(comment_rows) - # R3: LATEST onto the record itself. `row.update` too, so a LATER action in - # this same flow sees the enriched values — the rule `update_record` already - # follows, and without it "enrich, then route by follower count" would judge - # the record on the values it had before the pull. - # ⭐⭐ WAVE 31 · T39(c) — D-177(c): THE FIX APPLIED TO ONE PLATFORM AND NOT ITS - # TWIN, and that is the shape worth naming rather than the line. - # - # W30-T11 found this on TikTok the expensive way: a paid run enriched two - # profiles with `postMetrics` ON, wrote no posts, and said NOTHING — because - # `partial` takes THIS branch and only the `else` ever appended a note, so the - # reason `pull_profile` carries in `res["note"]` was discarded. Instagram has - # the identical branch and never got the fix; the sibling arm below has carried - # it since wave 30. One defect, two networks, one of them repaired — which is - # exactly how the last four TikTok/Instagram divergences were found. - # - # ⛔ ONLY WHEN IT WAS ASKED FOR, and ⛔ NOT WHEN THE BATCH DEFERRED — both - # conditions copied deliberately from the TikTok arm rather than re-reasoned. - # With `postMetrics` off, a `partial` is the normal answer for every profile - # and reporting it would put one useless line per record in the run entry; a - # DEFERRAL is a paid snapshot already filed for collection, and reporting it as - # a miss teaches a person to re-run and buy it twice. - # ⚠ Instagram's deferral sinks are `enrich["pending"]` (post metrics) and - # `pend_prof` (the profile snapshot) — the two lists `pull_profile` appends - # into — where TikTok reads `res["deferredMedia"]`. Different signal, same - # question: did this record's work get filed rather than lost? - if (cfg.get("postMetrics") and not (res.get("posts") or []) - and len(enrich["pending"]) == _pend_before and not pend_prof): - _why = _s(res.get("note"), 300) - if _why: - enrich["notes"].append(f"@{handle_raw}: {_why}") - cells = preset_cells(res, pulled) - if cfg.get("dryRun"): - enrich["dry"] = True - else: - row.update(cells) - _act_row_patch(patches, table, rid, cells) - else: - enrich["blocked"] += 1 - # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH. - # It used to be `_s(note, 90)` with no handle attached: three runs blocked - # the same record and the store could not say which record, let alone why. - # 90 characters also truncated the one measured sentence mid-clause. This is - # the most valuable thing the run produces and it now reaches the run entry. - enrich["notes"].append( - f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}") - # A vendor STATING that the account does not exist is remembered, so the next - # run stops paying to be told the same thing. - if res.get("gone"): - enrich["gone"][_gone_key(row, handle_raw)] = { - "at": _iso(), "handle": handle_raw, - "note": _s(res.get("note"), 300)} - elif kind == "enrich_tiktok": - # ⭐⭐ WAVE 30 · T08 (carrying wave-29's dropped T05). THE SECOND NETWORK, and it - # is a sibling of the branch above rather than a flag inside it. What is genuinely - # shared is shared by CALL — `enrich_selection`, `enrich_plan`, the pace floor, - # the tombstone memory; what differs is the four things B-9 measured as - # Instagram-hardcoded, and each has a TikTok counterpart of its own name. - pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"), - source=PROFILE_SOURCE_TT) - if not pkey: - # D-79, on the network that did not exist when D-79 was written: FAIL CLOSED, - # ONCE, WITH THE REASON — and the marker is its OWN, so a flow carrying both - # steps cannot report the Instagram sentence about the TikTok one. - if "unbound" not in tt["notes"]: - tt["notes"].append("unbound") - log("[aios-auto] enrich_tiktok: no profile column on " - f"{table}. Name one on the action, or mark a text column as a " - "TikTok profile") - continue - aid = str(act.get("id") or "") - if aid not in enrich_plan: - # ⚠ ONE plan dict for both networks is correct and not an oversight: it is - # keyed by ACTION id, and an action has exactly one kind. Two steps over one - # table are two budgets whichever networks they read. - chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone, - platform=PLATFORM_TIKTOK) - enrich_plan[aid] = set(chosen) - if sel_note: - tt["notes"].append(sel_note) - if str(rid) not in enrich_plan[aid]: - continue - handle_raw = str(row.get(pkey, "") or "").strip() - if not handle_raw: - continue # nothing to enrich on this record, not an error - # ⚠ THE PACE FLOOR IS THE VENDOR'S. There is no batch prefetch on this path yet - # (`pull_profile_tt` accepts one and nothing writes it — see the mailbox), so - # every profile after the first pays it. - if tt["profiles"]: - time.sleep(PACE_SECONDS) - tt["profiles"] += 1 - import connectors_tt as _tt_conn # lazy: connectors_tt imports this module - pend_prof = [] - res = _tt_conn.pull_profile_tt( - handle_raw, log=log, - pending_profile=(pend_prof if not cfg.get("dryRun") else None), - # ⭐ W30-T10. Both INCLUDE axes default OFF (W28/R5-R7), and the same - # `config` keys the Instagram step reads — one vocabulary, two networks. - max_posts=int(cfg.get("maxPosts") or 0), - post_metrics=bool(cfg.get("postMetrics")), - comment_metrics=bool(cfg.get("commentMetrics"))) - for _p in pend_prof: - _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() - # The deferred-profile queue is the Instagram one BY DESIGN: it is a vendor - # snapshot id waiting to be collected, and `_pending_profile_tasks` keys tasks by - # table+row, not by network. Sharing it is what makes the tick finish a TikTok - # read it has already been charged for. - enrich["pendingProfiles"].extend(pend_prof) - # ⭐⭐ W30 · D-156 — A DEFERRED MEDIA BATCH IS FILED, NOT NARRATED. Before this, - # a posts or comments scrape the vendor took too long over was reported in the - # note and collected by NOTHING: paid for, and recoverable only by a human reading - # a sentence. `connectors_tt` has already filtered these to the media corpora, so - # a profile snapshot cannot arrive here; what the engine adds is the vocabulary the - # QUEUE speaks — the `kind` (from the dataset id, which is one-to-one on TikTok) - # and the HANDLE, which the transport never knew. - # ⚠ `dryRun` queues NOTHING: a dry run buys nothing, so an entry here could only - # be a fixture leaking, and filing it would make the tick collect a snapshot that - # was never paid for. - if not cfg.get("dryRun"): - for _d in (res.get("deferredMedia") or []): - _kind = tt_metric_kind(_d.get("datasetId")) - if not _kind: - continue # not a media corpus ⇒ not this queue's business - tt["pending"].append({**_d, "kind": _kind, "requestedAt": _iso(), - "influencer": str(handle_raw).strip() - .lstrip("@").lower()}) - pulled = _iso() - if res["state"] in ("ok", "partial"): - tt["ok"] += 1 - snap_row = tt_snapshot_row(res, pulled) - if snap_row: - tt["snaps"].append(snap_row) - tt_idents, tt_metrics, tt_comments = tt_capture_rows(res, pulled) - tt["posts"].extend(tt_idents) - tt["psnaps"].extend(tt_metrics) - tt["comments"].extend(tt_comments) - # ⭐⭐ WAVE 30 · T11 — A CAPABILITY THAT WAS ASKED FOR AND DID NOT ARRIVE IS - # REPORTED. This is R6's second sentence applied to a capability rather than to - # a row cap, and it was found the expensive way: the 09:14 UTC paid run on - # nurilab enriched two profiles with `postMetrics` ON, wrote no posts, and said - # NOTHING — `ok: true`, no note, no count — because `partial` takes the branch - # ABOVE and only the `else` ever appended a note. Every `partial` return in - # `pull_profile_tt` carries the reason in `note` (*"this account's row carried - # no post links"*, *"the post source returned nothing"*, or the vendor's own - # words), and all of them were being discarded. A person then sees two enriched - # rows, an empty posts database and no explanation anywhere. - # ⛔ ONLY WHEN IT WAS ASKED FOR, which is the difference between a report and - # noise: with `postMetrics` off, `pull_profile_tt` returns `partial` + *"post - # capture is off for this step"* for EVERY profile, and appending that would put - # one useless line per record into the run entry and let the summary quote it. - # ⛔ AND NOT WHEN THE BATCH DEFERRED — a deferral is not a failure to deliver, - # it is a paid snapshot already filed for collection - # (`ttEnrichMetricBatchesPending`, D-156), and reporting it as a miss would - # teach a person to re-run and buy it twice. - if (cfg.get("postMetrics") and not (res.get("posts") or []) - and not (res.get("deferredMedia") or [])): - _why = _s(res.get("note"), 300) - if _why: - tt["notes"].append(f"@{handle_raw} (TikTok): {_why}") - cells = tt_preset_cells(res, pulled) - if cfg.get("dryRun"): - tt["dry"] = True - else: - row.update(cells) - _act_row_patch(patches, table, rid, cells) - else: - tt["blocked"] += 1 - # ⚠ TAGGED, because `run_notes` now carries BOTH networks' per-record reasons - # (the two flushes are concatenated). The summary picks ONE note to quote, so an - # untagged TikTok line could be quoted under Instagram's sentence and vice versa - # — which would undo the whole point of giving each network its own sentence. - tt["notes"].append( - f"@{handle_raw} (TikTok): {_s(res.get('note'), 300) or res['state']}") - # ⛔ AND THERE IS DELIBERATELY NO TOMBSTONE WRITER HERE, which is the opposite - # of an oversight. On the Instagram side `res["gone"]` has exactly ONE source — - # Apify answering `ACCOUNT_GONE_NOTE` (`connectors_ig`); Bright Data has no - # not-found verdict at all, and `DEFAULT_CHAINS["tt_profile"]` is deliberately - # single-provider, so nothing on this chain can say "no account exists". A - # phrase match invented against unmeasured vendor output would file a - # PERMANENT verdict (W28/R9 — no expiry) on the strength of a guess. - # ⚠ The READ side is still network-scoped above (`platform=PLATFORM_TIKTOK`) - # and that half is load-bearing today: `known_gone` is shared, Apify DOES - # write `instagram:` verdicts, and without the scoping one of those - # would silently suppress a TikTok read of a different person's account. - elif kind == "find_records": - found = find_records(rt, cfg.get("table"), cfg.get("cond"), - int(cfg.get("limit") or 25)) - counts["found"] += len(found) - # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had - # already stopped being reachable one wave earlier — which is the part worth reading. - # `_without_retired_board` strips every `review` action out of a definition on the - # READ path, so `all_definitions` and this function have not seen one since the board - # was retired. What was left behind was a branch referencing `skey`, `stamp` and - # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not - # dead code that merely wasted space: it was a `NameError` held back by a migration - # rather than by a guard, and any change that let one stored `review` action through - # would have crashed the whole action walk for every record in that flow. - # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps - # review "as an AI decision without lanes", and the cheap-first provider ladder behind - # it is real, working, measured work. They are PARKED, deliberately and in writing, - # not orphaned; see their own notes. - return False - - if actions: - for rid in list(row_ids or [])[:FLOOD_LIMIT]: - row = dict(rows.get(str(rid)) or {}) - if not row: - continue - # ⭐ D-191 — the web batch is built from THIS row's interpolated values, so it means - # nothing for the next one. Cleared here rather than inside `_walk`, which recurses - # into branches and would wipe a batch its own caller is still consuming. - web_done.clear() - _walk(actions, row, str(rid)) - # ⭐⭐ WAVE 30 · T08 — TWO FLUSHES, ONE `counts`, AND THE NOTES ARE CONCATENATED RATHER THAN - # OVERWRITTEN. `RUN_NOTES_KEY` is the one key both flushes legitimately produce, so a plain - # `counts.update(a); counts.update(b)` would drop every Instagram per-record reason the moment - # a flow also carried a TikTok step — silently, and precisely on the mixed flows where a - # person most needs to know which half failed. Every other key is prefixed and cannot collide. - _ig_out = _enrich_flush(rt, defn, username, enrich, log) - _tt_out = _tt_enrich_flush(rt, defn, username, tt, log) - _flush_notes = (list(_ig_out.pop(RUN_NOTES_KEY, None) or []) - + list(_tt_out.pop(RUN_NOTES_KEY, None) or [])) - counts.update(_ig_out) - counts.update(_tt_out) - if _flush_notes: - counts[RUN_NOTES_KEY] = _flush_notes - # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was - # incremented once per create the flow decided to make; what landed is what the store says, - # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches - # every row it made last time — which is the whole point of the feature). - counts.update(_commit_action_writes(rt, table, patches, creates, username, log)) - return counts - - -def migrate_field_instagram(rt, defn): - """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the - `enrich_instagram` action. Returns `(definition, changed)`. - - R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live - `field_instagram` automation to a plain flow carrying it; delete the kind and its label." - - ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration - turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** — - so a live automation whose `urlField` is blank has been working off that fallback for months. - The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a - blank `urlField` verbatim would produce an automation that USED to work and now refuses. The - fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit - binding — which is also the honest outcome: the column stops being implicit. - - ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The - old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s - column; the action writes the C1 PRESET CELLS instead. The status column is left in place and - simply stops being written — deleting somebody's column as part of a migration would be data - loss, and a stale cell beside a fresh `enriched_at` is readable for what it is. - - ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can - run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN - against the real stored definition" is meant to be satisfied. - """ - if (defn or {}).get("kind") != "field_instagram": - return defn, False - cfg = dict(defn.get("config") or {}) - target = str(cfg.get("targetTable") or "") - bound = str(cfg.get("urlField") or "").strip() - if not bound and target: - bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "") - try: - max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL) - except (TypeError, ValueError): - max_posts = 24 - act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None, - "config": {"profileField": bound, - "postMetrics": bool(cfg.get("postMetrics")), - "commentMetrics": bool(cfg.get("commentMetrics")), - "dryRun": bool(cfg.get("dryRun")), - "maxPosts": max(1, min(max_posts, 200))}} - flow = dict(defn.get("flow") or {}) - out = dict(defn) - out["kind"] = DEFAULT_KIND - # The enrich step goes FIRST — it is what the automation was, and every action the owner - # added afterwards was written expecting the capture to have happened. - out["flow"] = {"actions": [act] + list(flow.get("actions") or [])} - out["config"] = {"targetTable": target, - "targetLabel": str(cfg.get("targetLabel") or "")} - return out, True - - -def _enrich_flush(rt, defn, username, acc, log): - """C4: the enrich action's canonical IG tables, written ONCE for the whole run. - - Returns the counts the run reports. Empty when no enrich action ran, so a flow without one - pays nothing and its run history is unchanged (`run_now` merges only non-zero keys). - - ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three - tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then - the write-through to the platform master. Forking any of those would give the enrich action a - history that the metric fields could not see — and it is the LAST step, the write-through to - the platform master, that they read (W29-T34: `compute_metric_cells` → `ig_master.series_for`, - never this tenant's `ut_ig_snapshots`), so dropping that one line is the version of this fork - that would look harmless. - """ - if "unbound" in (acc.get("notes") or []): - # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step - # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so - # this function used to return `{}`, `run_now` merged only non-zero keys, and the run - # committed **`ok` — "N records walked"** having written nothing at all. The sentence - # naming the fix went to `log()`, i.e. the server console, which no customer reads. That - # is "green over nothing" on the one path where somebody is waiting for data. - # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a - # special case — and so the run entry itself records that this happened. - return {"enrichUnbound": 1} - # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail. - # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate - # was skipped as a known-dead handle — so the sentence explaining why the automation appears - # to do nothing would have been dropped on precisely the runs that most need it, and the - # owner would be back at "0 records walked, wtf". The selection note is produced before any - # profile is read and must outlive that early return. - if not acc.get("profiles"): - notes = list(acc.get("notes") or []) - return {RUN_NOTES_KEY: notes} if notes else {} - out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]} - if acc.get("notes"): - out[RUN_NOTES_KEY] = list(acc["notes"]) - # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so - # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel. - if not acc.get("dry"): - # The vendor's not-found verdicts, merged into engine state (never a tenant column and - # never a status string — W25/R4 retired those). Merged rather than replaced: a run that - # walked one record must not forget what earlier runs learned about the others. - if acc.get("gone"): - merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) - merged.update(acc["gone"]) - set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged}) - pending_profiles = queue_pending_profile_snapshots( - rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or []) - if pending_profiles: - out["enrichProfileBatchesPending"] = pending_profiles - queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), - acc.get("pending") or []) - if queued: - out["enrichMetricBatchesPending"] = queued - if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]): - # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. - if acc.get("dry"): - out["enrichDryRun"] = acc["profiles"] - return out - tag = str(defn.get("id") or "") - profile_table = str(_flow_table(defn) or "") - graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table) - snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], - graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) - missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key) - snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}), - acc["snaps"], "snapshot_key", cap=row_cap(snap_key)) - old_posts, collapsed_posts = dedupe_canonical_rows( - dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") - posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key)) - c_post["duplicates"] += collapsed_posts - psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), - acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key)) - old_comments, collapsed_comments = dedupe_canonical_rows( - dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") - comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key", - cap=row_cap(comment_key)) - c_comments["duplicates"] += collapsed_comments - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), - (comment_key, comments)): - if cur.get(k) is not None: - cur[k]["rows"] = rws - _refresh_relations_inplace(cur, log=log) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables - # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which - # is the failure a chart cannot show you. - capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"] - if capped: - out["enrichCapped"] = capped - log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap") - if missing: - out["enrichMissingTables"] = len(missing) - log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}") - out["enrichPoints"] = len(acc["snaps"]) - out["enrichPosts"] = c_post["inserted"] - out["enrichComments"] = c_comments["inserted"] - # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same - # contract `run_field_instagram` follows, because a pooled history with silent holes is worse - # than none. - try: - import ig_master - m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"], - acc["posts"], acc["psnaps"]) - if m_status == "error": - out["enrichMasterFailed"] = 1 - log(f"[aios-auto] enrich_instagram: the platform master copy FAILED. {m_note}; " - f"the tenant copy is complete and the next run re-appends") - elif m_status == "ok": - out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"]) - except Exception as e: # noqa: BLE001 - out["enrichMasterFailed"] = 1 - log(f"[aios-auto] enrich_instagram: master write-through raised " - f"{type(e).__name__}: {e}") - return out - - -#: How many candidate columns an unbound-enrich sentence names before it stops. Three, because the -#: sentence is read in a run log line: naming forty columns is the same as naming none. -UNBOUND_HINT_MAX = 3 - - -def _unbound_hint(rt, table_key): - """⭐ WAVE 30 · T14 — the *"…which column to mark"* half of D-79's sentence. - - Returns `" — this database's text columns are Handle, Who"` or `""`. Never a binding. - - ⛔ IT SUGGESTS AND DOES NOT RESOLVE, and the distinction is the whole reason this is a string - rather than a fallback. `profile_field_key`'s own docstring refuses to guess — *"the failure - would be a run that reports success having enriched from the wrong column … a wrong number is - harder to notice than a missing one"*. That argument is about BINDING. It says nothing against - telling a person, in the sentence they are already reading, which columns are even eligible: - the enrich still refuses, nothing is written, and the human makes the choice. - ⚠ `text` ONLY, matching the writer F shipped for the flag (`ColumnMenu`'s toggle is gated on - `editType === "text"`), so the sentence cannot offer a column the editor would then refuse. - """ - fields = (ut_get(rt, table_key) or {}).get("fields") or [] - # ⚠ AND A PRESET/LOCKED COLUMN IS NOT OFFERED. `Platform` is a `text` column on every preset - # profile table and carries `automation.preset` + an `editRole`, so the field editor refuses to - # retype it — offering it would send a person to a control that says no, which is exactly the - # claim the docstring above makes and did not honour on its first draft. - names = [str(f.get("label") or f.get("key") or "").strip() for f in fields - if str(f.get("type") or "text") == "text" - and not (f.get("automation") or {}).get("preset") - and not str(f.get("editRole") or "").strip() - and str(f.get("label") or f.get("key") or "").strip()] - if not names: - return "" - shown = ", ".join(names[:UNBOUND_HINT_MAX]) - more = len(names) - UNBOUND_HINT_MAX - return (f". This database's text columns are {shown}" - + (f" (+{more} more)" if more > 0 else "")) - - -def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=print): - """The four `ut_tt_*` tables, created-if-needed and written in ONE coalesced store update. - Returns `(inserted_by_table, capped, missing)`. - - ⭐⭐ WAVE 30 · D-156 — ONE WRITER, TWO CALLERS, AND THE SECOND CALLER IS WHY IT EXISTS. - This was the tail of `_tt_enrich_flush`, i.e. reachable only from an INLINE enrich. The - deferred collector needs exactly the same write, and the repo has already paid once for the - version where it did not: `top_up_views` lived inside `pull_profile_bd`, so it ran only when - the Posts scrape answered in time, and every DEFERRED Instagram run wrote posts with a blank - Views column. The fix there was this same shape — one function, two callers, so the inline and - deferred paths cannot answer differently — and copying the block instead would reintroduce the - class rather than the bug. - - ⭐ ONE LOOP OVER `(table, rows, key)` RATHER THAN FOUR HAND-WRITTEN BLOCKS, and the field list - comes from `TT_TABLE_FIELDS` — so a fifth `ut_tt_*` table is one tuple, and no table can be - created with a field list that disagrees with its own declaration. - ⚠ NO ROWS ⇒ NO DATABASE. Spawning `ut_tt_comments` on a collect that carried none gives a - person a database to watch never fill, which is the visible half of green-over-nothing. - """ - plan = [(TT_SNAPSHOTS_TABLE, snaps or [], "snapshot_key"), - (TT_POSTS_TABLE, posts or [], "shortcode"), - (TT_POST_SNAPSHOTS_TABLE, psnaps or [], "post_snapshot_key"), - (TT_COMMENTS_TABLE, comments or [], "comment_key")] - written, capped, missing = {}, 0, [] - inserted = {} - for table_key, rows_in, key_field in plan: - if not rows_in: - continue - # ⭐ R9 (W31-T32) — THE RUN PATH IS THE SITE THAT ACTUALLY CREATED TODAY'S TABLES, and it - # is the one the ticket's `how:` does not name. `waves/wave30/proof/tiktok-e2e-ut_tt_posts - # .png` shows `ut_tt_posts` with 8 real rows offering "+ New record" — those rows arrived - # HERE, not through the save path (which, until W31-T34, returned before the child spawn for - # a discovery automation). Stamping only the save site would have left every table that - # already exists unlocked. ⭐ And an EXISTING table does come forward on the next call: - # `ut_ensure`'s skip test carries `not (record_mode and have.get("recordMode") != - # record_mode)`, so no separate migration is needed. - real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username, - key=table_key, flow_tag=tag, lock_fields=True, - record_mode=tt_record_mode(table_key)) - missing.extend(ut_missing(rt, real_key)) - merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}), - rows_in, key_field, cap=row_cap(real_key)) - written[real_key] = merged - inserted[table_key] = counts_["inserted"] - capped += counts_["capped"] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for k, rws in written.items(): - if cur.get(k) is not None: - cur[k]["rows"] = rws - _refresh_relations_inplace(cur, log=log) - return cur - - if written: - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for every table - return inserted, capped, missing - - -def _tt_enrich_flush(rt, defn, username, acc, log): - """⭐⭐ WAVE 30 · T08 — the TikTok enrich action's append table, written ONCE for the whole run. - - The twin of `_enrich_flush` and deliberately NOT a call into it. That function is bound to - Instagram at four points — `ensure_ig_graph`, the four `IG_*_TABLE` keys, `PRESET_*` and the - write-through to the platform Instagram master — and the last of those is the one that would - look harmless: `ig_master.append_run` is what the metric FIELDS read (W29-T34), so a TikTok - creator's followers appended there would become an Instagram number in a pooled history that - no later run could tell apart. - - ⛔ EVERY KEY IT RETURNS IS PREFIXED `tt…`. `apply_actions` merges both flushes into one - `counts`, so `enriched`/`enrichBlocked` would be one name for two networks' numbers and a - mixed flow would report whichever flushed last. - - ⭐ WAVE 30 · T10 — FOUR TABLES NOW, AND EACH IS CREATED ONLY WHEN IT HAS A ROW. Spawning - `ut_tt_posts` on a run that captured no posts gives a person a database to watch never fill, - which is the visible half of green-over-nothing and the same complaint that produced - `discover_default_table` (*"a SECOND, empty database"*). So the create rides the rows. - """ - if "unbound" in (acc.get("notes") or []): - # D-79 on the second network: a COUNT, so `run_now`'s existing non-zero merge carries it - # and the run entry itself records that the step could not run. Its own key, so the - # summary can name TikTok rather than borrowing Instagram's sentence. - return {"ttEnrichUnbound": 1} - if not acc.get("profiles"): - # The selection's note outlives a run that read nothing — the same reason as the IG side: - # "every candidate was skipped" is exactly the run whose silence needs explaining. - notes = list(acc.get("notes") or []) - return {RUN_NOTES_KEY: notes} if notes else {} - out = {"ttEnriched": acc["ok"], "ttEnrichBlocked": acc["blocked"]} - if acc.get("notes"): - out[RUN_NOTES_KEY] = list(acc["notes"]) - # ⭐⭐ W30 · D-156 — QUEUED BEFORE THE DRY-RUN AND EMPTY-ROWS RETURNS BELOW, and the order is - # the point. A run whose media batch DEFERRED has no posts and no comments to write, so it - # takes the `not (snaps or posts or ...)` exit — the exact run that owns a paid snapshot id. - # Filing after that return would have collected nothing, forever, which is the shape the - # inline-vs-deferred split keeps producing. - # ⚠ Its own count key (`tt…`), like every other key here: `apply_actions` merges both - # networks' flushes into ONE dict, so sharing Instagram's name would let a mixed flow report - # one network's pending batches under the other's. - if not acc.get("dry"): - queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), - acc.get("pending") or []) - if queued: - out["ttEnrichMetricBatchesPending"] = queued - if acc.get("dry") or not (acc.get("snaps") or acc.get("posts") or acc.get("psnaps") - or acc.get("comments")): - # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. - if acc.get("dry"): - out["ttEnrichDryRun"] = acc["profiles"] - return out - tag = str(defn.get("id") or "") - inserted, capped, missing = _tt_write_tables( - rt, tag, username, acc.get("snaps"), acc.get("posts"), acc.get("psnaps"), - acc.get("comments"), log) - # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which - # is the failure a chart cannot show you. - if capped: - out["ttEnrichCapped"] = capped - log(f"[aios-auto] enrich_tiktok: {capped} row(s) refused by a table's row cap") - if missing: - out["ttEnrichMissingTables"] = len(missing) - log(f"[aios-auto] enrich_tiktok: could not create {', '.join(missing)}") - out["ttEnrichPoints"] = len(acc.get("snaps") or []) - if inserted.get(TT_POSTS_TABLE): - out["ttEnrichPosts"] = inserted[TT_POSTS_TABLE] - if inserted.get(TT_COMMENTS_TABLE): - out["ttEnrichComments"] = inserted[TT_COMMENTS_TABLE] - # ⛔ AND NO PLATFORM-MASTER WRITE-THROUGH, which is a deliberate absence rather than a missing - # line. `ig_master` is Instagram's pooled history and there is no TikTok equivalent yet; the - # tenant's own `ut_tt_snapshots` is the complete record today, and inventing a second store - # for a series nobody reads would be the fork this function exists to avoid. - return out - - -_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS)) -_PENDING_METRIC_KINDS = frozenset(("posts", "comments")) - -#: ⭐⭐ WAVE 30 · D-156 — THE TIKTOK HALF OF THE METRIC QUEUE, BUILT LAZILY FROM `connectors_tt`'s -#: OWN CONSTANTS. Re-declaring the two ids here would be a second copy of a vendor identifier that -#: nothing compares — the drift `MAX_UT_ROWS` demonstrated at 12x — so this reads them from the one -#: module that owns them, through `_tt_module()` because `connectors_tt` imports THIS module. -#: ⚠ It is a MAP rather than a set because on TikTok one dataset is exactly one kind, which is why -#: this network needs no `_tag_metric_deferrals` twin: the id the transport already recorded says -#: whether a batch is posts or comments, so nothing downstream has to be told twice. -_TT_METRIC_KIND_BY_DATASET = None - - -def tt_metric_kind(dataset_id): - """`"posts"` / `"comments"` for a TikTok media dataset id; `""` for anything else. - - ⛔ THE `""` IS LOAD-BEARING AND IS THE PLATFORM TEST. `collect_pending_metric_snapshots` - branches on it, so a dataset this map does not know keeps Instagram's mappers — which is the - safe direction, because Instagram's are what every stored pre-wave-30 task needs. - """ - global _TT_METRIC_KIND_BY_DATASET - if _TT_METRIC_KIND_BY_DATASET is None: - _tt = _tt_module() - _TT_METRIC_KIND_BY_DATASET = {str(_tt.TT_DS_POSTS): "posts", - str(_tt.TT_DS_COMMENTS): "comments"} - return _TT_METRIC_KIND_BY_DATASET.get(str(dataset_id or ""), "") - - -def _pending_metric_tasks(defn): - """Read validated, deduplicated paid metric snapshots from automation continuation state. - - A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally - stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or - the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor - endpoint, and the collector never starts a second scrape request. - """ - raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or [] - out, seen = [], set() - for item in raw if isinstance(raw, list) else []: - if not isinstance(item, dict): - continue - sid = str(item.get("snapshotId") or "").strip() - dataset = str(item.get("datasetId") or "").strip() - kind = str(item.get("kind") or "").strip() - handle = str(item.get("influencer") or "").strip().lstrip("@").lower() - # ⭐ W30 · D-156 — BOTH NETWORKS' MEDIA CORPORA ARE COLLECTABLE NOW. The membership test - # stays a WHITELIST (a snapshot id is a vendor capability that has already been paid for; - # accepting an unknown dataset would send our key at a corpus no mapper here can read), - # and TikTok's half is asked of the map that owns it rather than listed again. - if (not sid.startswith("sd_") - or (dataset not in _PENDING_METRIC_DATASETS and not tt_metric_kind(dataset)) - or kind not in _PENDING_METRIC_KINDS or not handle): - continue - key = (sid, dataset, kind, handle) - if key in seen: - continue - seen.add(key) - out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind, - "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""), - "lastChecked": str(item.get("lastChecked") or ""), - "lastNote": _s(item.get("lastNote"), 160)}) - return out - - -def queue_pending_metric_snapshots(rt, auto_id, pending): - """Durably retain new engagement snapshots without creating another paid provider request.""" - aid = str(auto_id or "").strip() - if not aid: - return 0 - incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}}) - if not incoming: - return 0 - added = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - definition = cur.get(aid) - if not isinstance(definition, dict): - return cur - state = definition.setdefault("state", {}) - existing = _pending_metric_tasks(definition) - known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"]) - for x in existing} - for task in incoming: - key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"]) - if key not in known: - existing.append(task) - known.add(key) - added[0] += 1 - state["pendingMetricSnapshots"] = existing - return cur - - _store_update(rt, _up, flush="sync") - return added[0] - - -def _match_profile_row(rows, handle): - """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there. - - ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With - `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and - taking the first row would write ONE profile's followers onto all of them — a wrong number that - looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the - same field `_bd_profile` reads first. - - ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points - at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that - shape the task's own handle is still the best evidence. It applies only when the snapshot holds - ONE row, so it can never mis-assign inside a batch. - """ - want = str(handle or "").strip().lstrip("@").lower() - usable = [n for n in (rows or []) if isinstance(n, dict)] - for node in usable: - got = str(_first(node, "account", "username", default="") or "").strip() - if got.lstrip("@").lower() == want and want: - return node - if len(usable) == 1: - return usable[0] - return None - - -def _pending_profile_tasks(defn): - """Validated, deduplicated deferred PROFILE snapshots from automation state. - - ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather - than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and - `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to - zero by its own validator, and even if it survived, the collector would hand it to - `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile - row. Reusing the name would have shipped a green no-op of exactly the class this change - exists to remove. - - ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written - back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be - resolved from the handle alone: two databases may both hold `@x`. - """ - raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or [] - out, seen = [], set() - for item in raw if isinstance(raw, list) else []: - if not isinstance(item, dict): - continue - sid = str(item.get("snapshotId") or "").strip() - dataset = str(item.get("datasetId") or "").strip() - handle = str(item.get("influencer") or "").strip().lstrip("@").lower() - table = str(item.get("table") or "").strip() - row_id = str(item.get("rowId") or "").strip() - # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a - # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor - # endpoint at all. - if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle - or not table or not row_id): - continue - if sid in seen: - continue - seen.add(sid) - out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile", - "influencer": handle, "table": table, "rowId": row_id, - "requestedAt": str(item.get("requestedAt") or ""), - "lastChecked": str(item.get("lastChecked") or ""), - "lastNote": _s(item.get("lastNote"), 200)}) - return out - - -def queue_pending_profile_snapshots(rt, auto_id, pending): - """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many - were newly added.""" - aid = str(auto_id or "").strip() - if not aid: - return 0 - incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}}) - if not incoming: - return 0 - added = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - definition = cur.get(aid) - if not isinstance(definition, dict): - return cur - state = definition.setdefault("state", {}) - existing = _pending_profile_tasks(definition) - known = {x["snapshotId"] for x in existing} - for task in incoming: - if task["snapshotId"] not in known: - existing.append(task) - known.add(task["snapshotId"]) - added[0] += 1 - state["pendingProfileSnapshots"] = existing - return cur - - _store_update(rt, _up, flush="sync") - return added[0] - - -def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step): - """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape. - - ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred` - list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the - vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown - away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a - 180 s budget, two abandoned `sd_…` snapshots in two runs. - - ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same - forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero - records ends the task with the vendor's reason attached — and, when the vendor blames the - target rather than itself, records the not-found verdict so the handle stops being re-bought. - """ - tasks = _pending_profile_tasks(defn) - if not tasks: - return ("ok", "No pending profile reads.", {}, [], {"source": "idle"}) - step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}") - remaining, patches, affected, notes = [], {}, [], [] - acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], - "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0, - "notes": [], "dry": False} - ready = waiting = closed = 0 - # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME - # `snapshotId`, and asking the vendor about it once per task would spend the round trips the - # batching just saved. Memoised for this collection pass only — a snapshot's state must be - # re-read on the NEXT run, which is a fresh call and a fresh dict. - _progress_memo, _rows_memo = {}, {} - - def _progress_of(sid): - if sid not in _progress_memo: - _progress_memo[sid] = bd_snapshot_progress(sid) - return _progress_memo[sid] - - def _store_profile(task, profile, via): - """Write ONE collected profile — snapshot row + preset cells on the record that asked. - - ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the - two paths cannot drift into writing different things — the same rule `top_up_views` - follows for the inline/deferred split. - """ - res = {"state": "ok", "profile": profile, "posts": [], "comments": [], - "via": via, "note": ""} - pulled = _iso() - snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) - acc["snaps"].append(snap_row) - acc["posts"].extend(idents) - acc["psnaps"].extend(metric_rows) - acc["comments"].extend(comment_rows) - _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled)) - affected.append(task["rowId"]) - - def _backup_profile(handle): - """The second rung, asked ONLY once the first has finished and delivered nothing. - - ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS - CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but - a profile that DEFERRED never came back through `pull_profile` — it came back here, and - this collector had no second rung at all. So the one path where the primary is most - likely to have failed was the one path with no fallback. - - ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary - answered is never re-bought from a second vendor — the mistake the capability split exists - to prevent. A refusal returns None and the caller reports blocked exactly as before. - """ - try: - import providers as _p - if not _p.PROVIDERS["apify"].can("ig_profile"): - return None - prof, note = apify_profile(str(handle)) - except Exception as exc: # noqa: BLE001 — a backup must not raise - log(f"[aios-auto] backup profile rung failed for @{handle}: " - f"{type(exc).__name__}: {exc}") - return None - if prof and prof.get("followers") is not None: - return prof - return None - - def _rows_of(sid): - if sid not in _rows_memo: - payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) - got = _bd_rows(payload) if not err else [] - if got and (_bd_deferral(payload) or - (len(got) == 1 and str(got[0].get("status") or "") in - ("running", "building", "collecting"))): - got = [] - _rows_memo[sid] = got - return _rows_memo[sid] - - for task in tasks: - stale_h = _hours_since(task.get("requestedAt")) - state, records, empty_note = _progress_of(task["snapshotId"]) - if state in ("done", "failed") and not records: - acc["profiles"] += 1 - # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing, - # so there is nothing left to wait for and no risk of buying the same record twice. - backup = _backup_profile(task["influencer"]) - if backup is not None: - ready += 1 - acc["ok"] += 1 - _store_profile(task, backup, "apify") - note = (f"@{task['influencer']}: the primary source returned nothing, so a backup " - f"source supplied the profile ({backup.get('followers')} followers)") - notes.append(note) - acc["notes"].append(note) - continue - closed += 1 - acc["blocked"] += 1 - note = f"@{task['influencer']}: {_s(empty_note, 240)}" - notes.append(note) - acc["notes"].append(note) - # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a - # profile request that means the account could not be reached at all, so the verdict - # is remembered and the selection stops re-buying it (R9: permanently, until a - # human edits the handle cell — see `clear_gone`). - # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about - # the query, and turning it into "this account does not exist" would silently retire - # live handles. - if state == "failed": - acc["gone"][_gone_key({}, task["influencer"])] = { - "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)} - continue - rows = _rows_of(task["snapshotId"]) if state != "running" else [] - if not rows: - # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of - # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never - # silently. An unbounded queue is the forever-loop this change removes, inverted. - if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS: - closed += 1 - note = (f"@{task['influencer']}: the source never finished the profile read " - f"queued {int(stale_h)}h ago ({task['snapshotId']}). It was dropped; " - f"the next run will ask again") - notes.append(note) - acc["notes"].append(note) - continue - waiting += 1 - remaining.append({**task, "lastChecked": _iso(), - "lastNote": _s("still building", 200)}) - continue - # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot - # holds several profiles and `rows[0]` would write one creator's numbers onto every record - # in the chunk. - node = _match_profile_row(rows, task["influencer"]) - if node is None: - closed += 1 - acc["profiles"] += 1 - acc["blocked"] += 1 - note = (f"@{task['influencer']}: the source delivered {len(rows)} profile" - f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle. " - f"it was dropped from the batch and the next run will ask again") - notes.append(note) - acc["notes"].append(note) - continue - ready += 1 - acc["profiles"] += 1 - profile = _bd_profile(node, task["influencer"]) - if profile.get("followers") is None and profile.get("following") is None: - acc["blocked"] += 1 - note = (f"@{task['influencer']}: the source delivered the profile but no " - f"follower/following counts were readable in it") - notes.append(note) - acc["notes"].append(note) - continue - acc["ok"] += 1 - _store_profile(task, profile, "brightdata:deferred") - notes.append(f"@{task['influencer']}: collected the profile the source had already been " - f"paid for ({profile.get('followers')} followers)") - - set_state(rt, str(defn.get("id") or ""), - {"pendingProfileSnapshots": remaining or None}) - counts = _enrich_flush(rt, defn, username, acc, log) - counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below - counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {}, - username, log)) - counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting, - "profileBatchesEmpty": closed}) - if notes: - counts[RUN_NOTES_KEY] = notes - head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected" - if ready else "no deferred profile read was ready") - tail = "".join([f"; {closed} finished with nothing to collect" if closed else "", - f"; {waiting} still building" if waiting else ""]) - state = "ok" if ready and not closed else "partial" - return (state, head + tail, counts, affected, - {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"}) - - -def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log): - """Write a completed metric snapshot through the canonical Post/Comment graph once.""" - if not (idents or snapshots or comments): - return {"posts": 0, "snapshots": 0, "comments": 0} - profile_table = str(_flow_table(defn) or "") - if not profile_table: - raise Refused("the pending engagement snapshot has no Profile database to link to") - graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), - profile_table=profile_table) - post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE], - graph[IG_COMMENTS_TABLE]) - old_posts, collapsed_posts = dedupe_canonical_rows( - dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") - posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key)) - c_post["duplicates"] += collapsed_posts - psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots, - "post_snapshot_key", cap=row_cap(ps_key)) - old_comments, collapsed_comments = dedupe_canonical_rows( - dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") - comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key", - cap=row_cap(comment_key)) - c_comments["duplicates"] += collapsed_comments - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)): - if cur.get(key) is not None: - cur[key]["rows"] = rows - _refresh_relations_inplace(cur, log=log) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - try: - import ig_master - status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots) - if status == "error": - log(f"[aios-auto] deferred post metrics master copy FAILED: {note}") - except Exception as exc: # noqa: BLE001 - log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}") - return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"], - "comments": c_comments["inserted"]} - - -def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step): - """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request.""" - tasks = _pending_metric_tasks(defn) - if not tasks: - return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"}) - step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}") - remaining, idents, snapshots, comments = [], [], [], [] - # ⭐ W30 · D-156 — TikTok's rows accumulate SEPARATELY and are written by TikTok's own writer. - # One queue can hold both networks' snapshots (they are keyed by dataset), but the two write - # paths address different tables and neither may touch the other's. - tt_collected = {"posts": [], "psnaps": [], "comments": []} - ready, waiting, closed, run_notes = 0, 0, 0, [] - for task in tasks: - # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be - # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued - # as "still building" on every tick — forever, because nothing about it would ever - # change. That is the same forever-loop the profile path was measured in, one dataset - # over, and it was latent here the whole time. - state, records, empty_note = bd_snapshot_progress(task["snapshotId"]) - if state in ("done", "failed") and not records: - # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect; - # keeping the entry would be a pending task that can never resolve. - closed += 1 - run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}") - continue - payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"}) - rows = _bd_rows(payload) if not note else [] - building = (state == "running" or not rows or _bd_deferral(payload) or - (len(rows) == 1 and str(rows[0].get("status") or "") in - ("running", "building", "collecting"))) - if building: - remaining.append({**task, "lastChecked": _iso(), - "lastNote": _s(note or "still building", 160)}) - waiting += 1 - continue - ready += 1 - pulled = _iso() - # ⭐⭐ WAVE 30 · D-156 — THE TIKTOK ARM. Before this, `_PENDING_METRIC_DATASETS` was - # Instagram's three, so a TikTok media batch the vendor deferred was paid for and - # collected by NOTHING — the note carried the snapshot id and a human was the only - # collector. It routes through `connectors_tt`'s own mappers and `tt_capture_rows`, - # never Instagram's, because every one of those stamps `PLATFORM_INSTAGRAM`. - # ⛔ TWO THINGS THE INSTAGRAM ARM DOES THAT THIS ONE MUST NOT, both deliberate: - # * NO `top_up_views`. TikTok's `play_count` arrives inline on the posts row, so there - # is no views capability to route to; calling it would buy an Instagram permalink. - # * NO `ig_master.append_run`. There is no TikTok master by design (`_tt_enrich_flush`), - # and a write-through would put a TikTok creator into Instagram's pooled history, - # which the metric FIELDS read — a wrong number in a permanent series. - if tt_metric_kind(task["datasetId"]): - _tt = _tt_module() - who = task["influencer"] - if task["kind"] == "posts": - mapped = [m for m in (_tt.normalize_post(r) for r in rows) if m] - for m in mapped: - # The snapshot was bought from THIS profile's own permalinks, so the backlink - # is a fact of the call even when a vendor row omits the author field. - m.setdefault("influencer_key", who) - res_tt = {"profile": {"handle": who}, "posts": mapped, "comments": []} - else: - mapped = [m for m in (_tt.normalize_comment(r) for r in rows) if m] - res_tt = {"profile": {"handle": who}, "posts": [], "comments": mapped} - tt_idents, tt_metrics, tt_comments = tt_capture_rows(res_tt, pulled) - tt_collected["posts"].extend(tt_idents) - tt_collected["psnaps"].extend(tt_metrics) - tt_collected["comments"].extend(tt_comments) - continue - if task["kind"] == "posts": - mapped_posts = [] - for raw in rows: - post = _bd_post_metrics(raw) - if post: - # The snapshot was created from this Profile's canonical post URLs. Keep - # that explicit backlink even when a vendor response omits `user_posted`. - post["influencer_key"] = task["influencer"] - mapped_posts.append(post) - if mapped_posts: - # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the - # owner's `theresalearns` run filled every column except Views. - # - # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a - # Posts row physically cannot carry a view count — MEASURED on the stored - # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them. - # Views only ever comes from the Apify capability. That top-up lived INSIDE - # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait - # budget; when the batch deferred — which is routine, and what happened here — the - # rows came back through THIS function and Apify was never asked. - # ⇒ `top_up_views` is now one function with two callers rather than a copy, so - # the inline and deferred paths cannot answer this differently again. - # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which - # is what freezes the values into the post + snapshot rows. - v_note = top_up_views(mapped_posts, log=log) - if v_note: - run_notes.append(f"@{task['influencer']}: {v_note}") - _unused, post_rows, metric_rows, embedded = capture_rows( - {"state": "ok", "profile": {"username": task["influencer"]}, - "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled) - idents.extend(post_rows) - snapshots.extend(metric_rows) - comments.extend(embedded) - else: - for raw in rows: - comment = _bd_comment(raw, influencer_key=task["influencer"]) - if comment: - comments.append(comment) - - set_state(rt, str(defn.get("id") or ""), - {"pendingMetricSnapshots": remaining or None}) - written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log) - counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting, - "metricBatchesEmpty": closed, - "postEngagementSnapshots": written["snapshots"], - "commentsCollected": written["comments"]} - # ⭐ W30 · D-156 — the TikTok write, through the SAME function the inline enrich uses, so a - # collected batch and an inline one cannot land differently. Its counts carry the `tt` prefix - # for the reason every other TikTok count does. - if any(tt_collected.values()): - tt_inserted, tt_capped, tt_missing = _tt_write_tables( - rt, str(defn.get("id") or ""), username, [], tt_collected["posts"], - tt_collected["psnaps"], tt_collected["comments"], log) - for key, name in ((TT_POSTS_TABLE, "ttPostsCollected"), - (TT_POST_SNAPSHOTS_TABLE, "ttPostSnapshotsCollected"), - (TT_COMMENTS_TABLE, "ttCommentsCollected")): - if tt_inserted.get(key): - counts[name] = tt_inserted[key] - if tt_capped: - counts["ttCollectCapped"] = tt_capped - log(f"[aios-auto] collect: {tt_capped} TikTok row(s) refused by a table's row cap") - if tt_missing: - log(f"[aios-auto] collect: could not create {', '.join(tt_missing)}") - # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved - # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four - # other call sites disagree about the shape of a run — and `_commit_run` already drops - # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather - # than to a crash. - if run_notes: - counts[RUN_NOTES_KEY] = run_notes - # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is - # a real outcome the tenant paid for and it must read as an answer, not as a disappearance. - tail = (f"; {closed} finished with nothing to collect" if closed else "") - if waiting: - return ("partial", - f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; " - f"{waiting} still building and will be collected automatically{tail}", - counts, [], {"capture_posts": "partial", "write": "ok"}) - return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}", - counts, [], {"capture_posts": "ok", "write": "ok"}) - - -def ai_decide(rt, defn, act, row, row_id="", log=print): - """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" - to leave the card for a person. - - ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed - answer, or a label the review does not offer all return "" — and "" means the card sits at - the review gate exactly as it would with no AI at all. The feature can be broken, absent or - wrong and the worst outcome is a human doing the work. - - ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN - ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board - under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely - worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter → - anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is - MISSING is only the door — an action kind that asks a question and takes an answer, without a - stage column to write it into. **Do not delete this in a dead-code sweep without reading that - sentence first**; equally, do not treat it as shipped — nothing reaches it today. - """ - cfg = act.get("config") or {} - options = list(cfg.get("next") or []) - if not options: - return "" - try: - import ai_review - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") - return "" - fields = [f.get("key") for f in - (ut_get(rt, ((defn.get("config") or {}).get("targetTable") - or (defn.get("trigger") or {}).get("table") or "")) or {} - ).get("fields") or []] - # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — ATTRIBUTE THE SPEND. `st=` is what lets the ledger write - # to the right tenant's store and `user=` is who it is billed to; without them the call is - # counted as UNATTRIBUTED, which is a meter that reports a total nobody can act on. - # ⚠ `createdBy` is the honest actor here: an AI review decision is made ON BEHALF of the - # automation, by a scheduler, with no person at the keyboard. Naming whoever last edited it - # would attribute a nightly run to an editor who was asleep. - choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, - row=row, fields=[f for f in fields if f], - label=cfg.get("label") or "Review", - st=rt, user=str(defn.get("createdBy") or "")) - if not choice: - if meta.get("problem"): - log(f"[aios-auto] ai review declined to answer: {meta['problem']}") - return "" - review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", - choice, meta.get("provider") or "ai", by="ai", - note=meta.get("reason") or "", model=meta.get("model") or "") - return choice - - -def find_records(rt, table_key, cond, limit=25): - """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts - the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" - rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} - out = [] - for rid, row in rows.items(): - if lane_match(cond, row or {}): - out.append(str(rid)) - if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): - break - return out - - -def _commit_action_writes(rt, table, patches, creates, username, log): - """The ONE write. Row patches merge into the target's rows; creates land in their own tables — - APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a). - - Returns the REALIZED create counts, and returning them is the point rather than a - convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only - this function knows how many of those became rows, how many matched one that was already - there, and how many the cap refused. A run that reported the attempt as "created" would be - the summary-disagrees-with-what-happened defect this module names in three other places. - - ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE - implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand - rolling a third inside this function is that debt returning with a new name — and it would - quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED, - NEVER DELETED, and `capped` is its own count rather than folded into `skipped`). - """ - out = {"created": 0, "createUpdated": 0, "createUnchanged": 0, - "createCapped": 0, "createSkipped": 0} - if not patches and not creates: - return out - # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one - # `ut_write_rows` per patched table, and that was free while the only patcher was a review - # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the - # step it reached, so the flow's own table is now patched on essentially every run — and a - # flow that also creates into that same table would have committed it TWICE per run, against - # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around. - # Staged into `staged` and handed to the creates pass, which writes each table exactly once. - staged = {} - for tkey, rowpatch in (patches or {}).items(): - cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) - for rid, vals in rowpatch.items(): - cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} - staged[str(tkey)] = cur - # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The - # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once - # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr - # repo budget this whole module is shaped around. - by_table = {} - for (tkey, unique), new_rows in (creates or {}).items(): - by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows)) - for tkey, groups in by_table.items(): - t = ut_get(rt, tkey) - if t is None: - n = sum(len(r) for _u, r in groups) - log(f"[aios-auto] create_record: {tkey} no longer exists. {n} skipped") - out["createSkipped"] += n - continue - # The PATCHED rows when this table was also stamped this run, so the creates land on top - # of the stamp rather than on a copy of the store that predates it. - cur = staged.pop(tkey, None) - cur = dict(t.get("rows") or {}) if cur is None else cur - cap = row_cap(tkey) - for unique, new_rows in groups: - if unique: - cur, c = upsert_rows(cur, new_rows, unique, cap=cap) - out["created"] += c["inserted"] - out["createUpdated"] += c["updated"] - out["createUnchanged"] += c["unchanged"] - out["createCapped"] += c["capped"] - # ⚠ `skipped` here means "this row had no value for the unique key", which for a - # create action is a mapped value that interpolated to nothing — worth surfacing, - # because the symptom is otherwise a run that says it created less than it walked. - out["createSkipped"] += c["skipped"] - if c["capped"]: - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " - f"{c['capped']} row(s) not written") - continue - nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 - for i, vals in enumerate(new_rows): - if len(cur) >= cap: - # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is - # named, here and in the run's counts. This used to `break` with a log line - # that said the cap was hit and never said how much was lost. - out["createCapped"] += len(new_rows) - i - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " - f"{len(new_rows) - i} row(s) not written") - break - cur[str(nxt)] = dict(vals) - nxt += 1 - out["created"] += 1 - ut_write_rows(rt, tkey, cur) - # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and - # once each, so "one store write per table" holds whether a table was patched, created into, - # or both. - for tkey, cur in staged.items(): - ut_write_rows(rt, tkey, cur) - return out - - -def _lane_sentence(cond, top=True): - """One condition tree → the sentence a step's `detail` carries on the canvas. - - C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a - label never claims a flat comparison the tree does not make. The SERVER composes it, for the - same reason it composes every other `detail` — a client paraphrase of a structure the engine - evaluates is a second implementation of the same sentence, free to drift from it. - - ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is - live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a - rename would be churn on a working function to fix a word. - """ - if cond is None: - return "Everything else" if top else "" - if isinstance(cond, dict): - for key, joiner in (("all", " and "), ("any", " or ")): - if key in cond: - parts = [_lane_sentence(c, False) for c in cond.get(key) or []] - parts = [p for p in parts if p] - if not parts: - return "" - inner = joiner.join(parts) - return inner if top or len(parts) == 1 else f"({inner})" - v = cond.get("value") - return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") - - -def _rid_num(rid): - return int(rid) if str(rid).isdigit() else 10 ** 9 - - -#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can -#: grow a definition without limit is a serialisation cost wearing a compliance hat. -MAX_REVIEWS = 100 - - -def review_audit(rt, auto_id, row_id, from_label, to_label, username, - by="user", note="", model=""): - """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to - the definition (newest first, bounded). - - ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that - made it and the one-line reason it gave. One audit log, not two — a reader asking "who - decided this card" must not have to know there are two places to look, and the moment an - AI decision is invisible beside a human one the log stops being an audit. - - ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone: - `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote - this shape off a stage field's `flowId` — a field the migration now drops. Kept because the - SHAPE is the contract a future decision action would write, and re-deriving an audit format - is how two of them end up existing. - """ - entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), - "from": _s(from_label, 60), "to": _s(to_label, 60), - "by": "ai" if by == "ai" else "user"} - if note: - entry["note"] = _s(note, 300) - if model: - entry["model"] = _s(model, 60) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] - return cur - - _store_update(rt, _up, flush="sync") - - -# --------------------------------------------------------------------------------------------- -# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) -# --------------------------------------------------------------------------------------------- -# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — -# the value reflects every pull the platform has, not just this tenant's). Computed by the -# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files -# no session owns this wave), human-write-refused at the grid door, and every number drills to -# the exact snapshot rows behind it. -# -# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no -# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero -# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not -# readable") — the `_bd_posts_count` law, one layer up. - -METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") -METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") -#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each -#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; -#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). -PROFILE_MEASURES = ("followers", "avg_engagement") -METRIC_AGGS = ("avg", "sum", "latest") - - -def metric_value(series, measure, window, agg="", today=None): - """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. - - `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference - day; post-count windows count back from the newest post. `drill_rows` are the exact rows - the number came from, so the route can honour [[no-unverifiable-aggregates]] without - recomputing differently.""" - series = series or {} - today = today or _now() - if measure in PROFILE_MEASURES: - rows = [r for r in series.get("snapshots") or [] - if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] - if window in ("last_7d", "last_30d"): - days = 7 if window == "last_7d" else 30 - floor = today - _dt.timedelta(days=days) - rows = [r for r in rows - if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] - if not rows: - return None, [] - if window == "latest" or agg == "latest": - picked = [rows[-1]] - else: - picked = rows - vals = [_lane_num(r.get(measure)) for r in picked] - vals = [v for v in vals if v is not None] - if not vals: - return None, [] - out = vals[-1] if (window == "latest" or agg == "latest") else \ - (sum(vals) if agg == "sum" else sum(vals) / len(vals)) - if measure == "avg_engagement": - # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs - # transform-pct scar) — scaled exactly once, here. - return f"{out * 100:.2f}", picked - return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked - # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. - posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] - posts.sort(key=lambda p: str(p.get("posted_at"))) - if window == "latest": - picked_posts = posts[-1:] - elif window == "last_3_posts": - picked_posts = posts[-3:] - else: - days = 7 if window == "last_7d" else 30 - floor = today - _dt.timedelta(days=days) - picked_posts = [p for p in posts - if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) - or _dt.datetime.min) >= floor] - vals, drill = [], [] - for p in picked_posts: - snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] - for snap in reversed(snaps): - raw = snap.get(measure) - v = _lane_num(raw) - if v is not None and str(raw).strip() != "": - vals.append(v) - drill.append(snap) - break - if not vals: - return None, [] - out = sum(vals) if (agg or "sum") == "sum" else \ - (vals[-1] if agg == "latest" else sum(vals) / len(vals)) - return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill - - -def metric_fields_of_table(t): - return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] - - -def _table_handle(row, url_field): - return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) - or "").strip().lower() - - -def compute_metric_cells(rt, table_key, today=None, tables=None, persist=False): - """Recompute every metric cell on ONE table from the master series. One coalesced write, - only when something actually changed (the flush-ceiling law); zero reads when the table - has no metric fields or the master is off. Returns the number of rows touched.""" - owned = tables is not None - blob = tables if owned else None - t = ((blob or {}).get(str(table_key)) if owned else ut_get(rt, table_key)) - mfields = metric_fields_of_table(t) - if not mfields: - return 0 - import ig_master - if not ig_master.configured(): - return 0 - url_field = next((f.get("key") for f in (t.get("fields") or []) - if f.get("type") == "url"), "") - rows = t.get("rows") or {} - handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} - series = ig_master.series_for({h for h in handles.values() if h}) - changes = {} - for rid, row in rows.items(): - s = series.get(handles.get(rid) or "") - for f in mfields: - bag = f["metric"] - val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), - bag.get("agg") or "", today=today) - if s else (None, [])) - want = "" if val is None else str(val) - if str((row or {}).get(f["key"], "")) != want: - changes.setdefault(str(rid), {})[f["key"]] = want - if not changes: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for rid, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) - return cur - - if owned: - _up(blob) - if persist: - rt.update(UT_STORE_KEY, _up, flush="sync") + # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds + # are dropped rather than carried — a link to a deleted row is not a link. + lrows = (linked.get("rows") or {}) + resolved[lk_key] = { + str(rid): [(i, lrows[i]) for i in + [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] + if i and i in lrows] + for rid, row in rows.items()} + + changes = {} + for rid, row in rows.items(): + rid = str(rid) + row = row or {} + for f in links: + fk = str(f["key"]) + # ⛔ A REFUSED LINK IS SKIPPED, NOT BLANKED — the difference between "this could not + # be resolved, here is why" and "there are no linked records". Its cell keeps the last + # value that WAS resolvable; the refusal rides out in `limits`. + if fk in refused: + continue + hits = (resolved.get(fk) or {}).get(rid) or [] + if f["link"].get("single"): + hits = hits[:1] + # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell + # is a projection of `resolved`, which is uncapped and is what every rollup below + # consumes — so a profile with 900 posts shows the first 500 ids and still averages + # over all 900. Same argument the `posts` window already makes: shedding is safe + # precisely because the authoritative store still holds everything. + want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + for f in rollups: + fk, bag = str(f["key"]), f["rollup"] + lk_key = str(bag.get("link") or "") + # Same law one loop up: a rollup whose LINK was refused folds nothing and is left + # alone, rather than printing a 0 that reads as a measurement. + if lk_key in refused: + continue + hits = list((resolved.get(lk_key) or {}).get(rid) or []) + # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING + # and therefore to a blank cell — never to a stale number. A column that keeps + # printing yesterday's answer after its input is gone is the worst of the options. + # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE + # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so + # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* + # rather than *the ones over X among the last 10* — two different windows wearing one + # sentence. Harmless while a threshold was a literal; incoherent the moment a + # threshold is a statistic OF the window, because the set being described and the set + # doing the describing would be different sets. + # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s + # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is + # computed over that window, THEN the conditions filter it") and the validator half + # was written against it. + # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH + # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` + # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` + # with no limit), so the blast radius is user-built rollups only. + # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last + # 12 posts, we also want to make it so its last N record, where the record's Status is + # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, + # so they select among the rows the window already kept. `where` selects WHICH rows + # the window is spent on. + # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per + # identity; run it first and a carousel could claim the slot its reel sibling needed, + # so the window would come up short for a reason nothing on screen explains. + # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this + # list, because at this point there is no fixed set for a statistic to be about. + where = list(bag.get("where") or []) + if where: + where_matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=None) + for condition in where] + if str(bag.get("whereConj") or "and") == "or": + hits = [pair for pair in hits if any(where_matches(pair))] + else: + hits = [pair for pair in hits if all(where_matches(pair))] + sort_by = str(bag.get("sortBy") or "") + if sort_by: + ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") + # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not + # rankable, and it must land at the END whichever direction is asked for — which + # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel + # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its + # window on rows that HAVE the value before it falls back to ones that do not. + keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] + rankable = [(p, k) for p, k in keyed if k is not None] + rankable.sort(key=lambda pk: pk[1], + reverse=str(bag.get("sortDir") or "desc") == "desc") + hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] + distinct_by = str(bag.get("distinctBy") or "") + if distinct_by: + seen, unique = set(), [] + for pair in hits: + identity = str(pair[1].get(distinct_by) or "").strip().lower() + # A blank is not an identity. Keep it rather than collapsing every unknown + # record into one synthetic duplicate. + if identity and identity in seen: + continue + if identity: + seen.add(identity) + unique.append(pair) + hits = unique + limit = int(bag.get("limit") or 0) + if limit: + hits = hits[:limit] + # --- the window is now FIXED, so a set-statistic threshold has a set to be about. + conditions = list(bag.get("conditions") or []) + if conditions: + # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the + # window; computing it inside `matches` would recompute the same mean for every + # candidate and — worse — would invite computing it over a set that the filter is + # already shrinking underneath it. + # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond + # the mean") is falsy, and the `or` spelling would silently rewrite it to the same + # number by accident. It reads identically and is right for the wrong reason, + # which is how it survives a review. + refs = [_rollup_ref_threshold( + [r for _i, r in hits], str(c.get("field") or ""), + (c.get("ref") or {}).get("sigmas", 0.0)) + if isinstance(c, dict) and c.get("ref") is not None else None + for c in conditions] + matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=ref) + for condition, ref in zip(conditions, refs)] + if str(bag.get("conditionConj") or "and") == "or": + hits = [pair for pair in hits if any(matches(pair))] + else: + hits = [pair for pair in hits if all(matches(pair))] + src = str(bag.get("field") or "") + # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the + # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at + # all, so the default stands for it. + want = _rollup_fold(str(bag.get("fn") or ""), + [r.get(src) for _i, r in hits] if src else [1] * len(hits), + (linked_types.get(lk_key) or {}).get(src, "text")) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + return changes, limits + + +def compute_relation_cells(rt, table_key, tables=None): + """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. + + Zero store reads when the table declares neither kind — the same cheap-by-construction shape + `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. + + ⭐ W41-T18 — THE PERSISTING WRAPPER OVER `relation_cells`, WHICH IS WHERE THE WORK MOVED. A + caller that wants the cells themselves (the grid render for a registry module, the Relational + pivot) calls that one and reads BOTH halves of its answer; this one exists for the tick, which + only ever wanted the count. + + ⛔ A REGISTRY MODULE PERSISTS NOTHING HERE, AND THE COUNT IS STILL TRUE. `customer_data` and + `product_data` store no rows in the `user_tables` document — their grids are assembled per + render from the tenant's Odoo pool — so there is nothing on disk for a derived cell to update + and `_up` below would find no table to write into. The count reports cells RESOLVED, and the + cells themselves are what `relation_cells` hands back; a caller that needs them must call it. + + ⛔⛔ THE SIGNATURE IS FROZEN, AND IT COST A GATE RUN TO LEARN WHY. A first cut added a + `log=None` kwarg here so the tick could print a refusal. `verify_automation`'s NC55 WRAPS this + function (`newest_wins(rt, table_key, tables=None)`) — as any wrapper reasonably would — and + the new keyword made every call a `TypeError`. `_refresh_relations_inplace` CATCHES every + exception and logs it, so the whole relational pass went dead with nothing red on the main + run: 2224/2224 still passed and only the negative-control sweep noticed, two checks deep in + another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the + caller that renders them reads them, and nothing about this door changes shape. + """ + changes, _limits = relation_cells(rt, table_key, tables=tables) + if not changes: + return 0 + if str(table_key) in LINKABLE_MODULES: + return len(changes) + + # A run that just wrote linked rows passes its in-flight user_tables bucket here so the + # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps + # the public helper's old persist-on-change behaviour. + if tables is not None: + tt = tables.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) return len(changes) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") return len(changes) - - -# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ──────────────── -# -# Owner: *"adding relational database function with links and rollups… exactly like how Airtable -# does it… and this rollup needs to have formula that we can use to calculate things like average -# Views over last N posts."* -# -# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula -# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not -# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced -# write only when something changed — and inherits its flush-ceiling discipline for free. -# -# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching -# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is -# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as -# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED -# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The -# moment one writes a number nothing else can re-derive, it has become a third copy. - -def _ut(): - """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic. - - Pulling this module in at import time initialises the store layer earlier than - `automation_engine` used to, and the last time that happened it moved a store-commit COUNT - from three to four on an unrelated gate. The relational pass - needs three constants and two predicates from the field layer; it does not need to change - when this module is imported. - """ - import core.user_tables as _m - return _m - - -#: The fns by family — how a value is folded, and what a blank means in each. -_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"}) -_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"}) -#: The count/order family — folded by their own arms above the lanes. -_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"}) -#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how -#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED -#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm -#: claims fell past both lanes into the join below — so a "Std deviation" column rendered -#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now -#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question -#: nothing can answer; a comma-joined list is a different question's answer wearing this label). -_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) -#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED. -#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name -#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot -#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the -#: defect above, because the defect was that the two lists already disagreed. -ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS - | _ROLLUP_TEXT_FNS) -_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) -#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing. -_ROLLUP_JOIN = ", " -#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a -#: cell nobody can read is not an aggregate. -ROLLUP_MAX_CHARS = 4000 - - -def _sort_key(value, ftype): - """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell - is blank or does not parse as its declared type. - - ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a - string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides - WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently - averages the wrong twelve posts. - - ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION - SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")` - otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True` - flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent - real one out of the window. A sentinel inside the key cannot mean "last" in both directions, - because the direction is applied to the sentinel too. - ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as - `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at` - only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every - key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed - because its fixture posts all carried dates. - ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it - sorts, and "an unknown date is not a recent one" is finally what the code does. - """ - raw = "" if value is None else str(value).strip() - if raw == "": - return None - if ftype in ("int", "currency", "pct", "rating"): - n = _lane_num(raw) - return (n,) if n is not None else None - if ftype == "date": - d = _parse_iso(raw.replace(" ", "T")) - return (d.timestamp(),) if d is not None else None - return (raw.lower(),) - - -def _sample_stdev(nums): - """SAMPLE standard deviation (n-1) of `nums`, or None under two values. - - ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a - function: `_rollup_fold` RENDERS this number into a "Std deviation" column and - `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy - would let a column and the filter beside it disagree about the same word on the same set — - the exact drift `top_up_views` was extracted to prevent one module over - ([[one-evaluator-per-question]]). - ⛔ None means UNANSWERABLE and no caller may read it as 0. - """ - if len(nums) < 2: - return None - mean = sum(nums) / len(nums) - return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5 - - -def _rollup_ref_threshold(rows, field, sigmas): - """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1). - - ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`, - and NOT yet filtered by the conditions this threshold feeds. That order is the contract - (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders - produce a plausible number: computing it before `limit` answers "2 sigma of everything this - account ever posted" under a column that says "of the last 10", and computing it after the - filter makes the threshold depend on the rows it is choosing — a definition that chases - itself. - - Returns None when the window cannot answer — fewer than two numeric values in `field`. - ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question - with a permissive answer; letting an unanswerable leaf pass everything would silently turn - "the outliers" into "all of them", which is this module's worst failure mode wearing a filter. - """ - nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None] - sd = _sample_stdev(nums) - if sd is None: - return None - return (sum(nums) / len(nums)) + float(sigmas) * sd - - -def _rollup_fold(fn, values, ftype="text"): - """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING. - - Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's - oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a - question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement - that the creator gets no views. - ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records" - over an empty set is genuinely 0, not unknown. - - `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only - `min`/`max` read it today; it defaults to `text` so every existing caller and every test that - folds a bare list keeps its exact previous answer. - """ - if fn == "countall": - return str(len(values)) - if fn == "latest": - # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve - # a blank on the newest row rather than reaching backwards and presenting an older value - # as current. - return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS] - if fn == "counta": - return str(len([v for v in values if str(v or "").strip() != ""])) - if fn == "count": - # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them - # distinct is the whole reason both exist. - return str(len([v for v in values if _lane_num(v) is not None])) - # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN. - # - # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered - # BLANK forever while looking completely configured — the failure this module refuses - # everywhere else, arriving through the one fold that had no type awareness. "Earliest order" - # and "latest invoice due" are the two most ordinary date rollups there are, and - # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates - # and the fold did not. - # - # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a - # string-compare fallback would have worked on well-formed data and silently mis-ordered a - # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is - # already at this call site (`linked_types`), and `_sort_key` is already the one function that - # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather - # than growing a second idea of what a date is. - # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person - # wants in the column is the stored date string exactly as the source row spells it. - # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean - # of two timestamps is a number this product has no column type for, and inventing one here - # would be a value with no author. - if fn in ("min", "max") and ftype == "date": - keyed = [(k, str(v)) for k, v in - ((_sort_key(v, "date"), v) for v in values) if k is not None] - if not keyed: - return "" - return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS] - if fn in _ROLLUP_NUM_FNS: - nums = [n for n in (_lane_num(v) for v in values) if n is not None] - if not nums: - return "" - if fn == "stdev": - # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference - # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to - # be LINKED, which is a sample of an account's posting history and not its entirety. - # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION - # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently - # switched divisors. - # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the - # honest reason is upstream of the arithmetic: one measurement has no spread to - # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the - # single most confident thing this column can say, asserted from a single row. Same - # law as the blank `sum`, and it bites harder here. - out = _sample_stdev(nums) - if out is None: - return "" - else: - out = (sum(nums) if fn == "sum" else min(nums) if fn == "min" - else max(nums) if fn == "max" else sum(nums) / len(nums)) - return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}" - if fn in _ROLLUP_BOOL_FNS: - # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero". - flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values] - if not flags: - return "" - hit = (all(flags) if fn == "and" else any(flags) if fn == "or" - else sum(1 for f in flags if f) % 2 == 1) - return "1" if hit else "" - # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`. - # An fn no arm above recognises returns "" rather than a comma-joined dump of every value, - # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28. - if fn not in _ROLLUP_TEXT_FNS: - return "" - vals = [str(v).strip() for v in values if str(v or "").strip() != ""] - if fn == "arrayunique": - seen, uniq = set(), [] - for v in vals: - if v.lower() not in seen: - seen.add(v.lower()) - uniq.append(v) - vals = uniq - if not vals: - return "" - text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals)) - return text[:ROLLUP_MAX_CHARS] - - -def _link_from_key(fields, bag): - """Which column on THIS table supplies the join value. - - Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That - fallback chain is what makes an Instagram database link up with no configuration at all — - the "automatically" in the owner's instruction — and it is the same chain the grid already - uses to decide a table's identity column, rather than a second opinion about it. - """ - declared = str((bag or {}).get("from") or "").strip() - if declared: - return declared - prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None) - if prof: - return str(prof.get("key") or "") - pin = next((f for f in fields if f.get("pinned") is True), None) - return str(pin.get("key") or "") if pin else "" - - -def _linked_rows_by_join(linked, on_key): - """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE. - - ⚠ Lower-cased because the join values this exists for are Instagram handles, which the - profile flag already normalises to lower case on one side and which a hand-typed cell on the - other side may not. A join that misses on case is a relation that silently reports zero. - """ - idx = {} - for rid, row in ((linked or {}).get("rows") or {}).items(): - k = str((row or {}).get(on_key) or "").strip().lower() - if k: - idx.setdefault(k, []).append((str(rid), row or {})) - return idx - - -def _rollup_condition_matches(row, condition, field_types, ref_value=None): - """Evaluate one Airtable-style linked-record condition against a candidate row. - - `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the - caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed - here because this function sees one row and the statistic is a property of the whole window — - a version that reached for the set from inside would be recomputing the same mean once per - row, and would have to be handed the window anyway. - ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the - threshold helper for why the permissive reading is the dangerous one. - """ - field = str((condition or {}).get("field") or "") - op = str((condition or {}).get("op") or "") - raw = (row or {}).get(field) - text = str(raw or "").strip() - if op == "is_empty": - return text == "" - if op == "is_not_empty": - return text != "" - if (condition or {}).get("ref") is not None: - # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering - # ops, and a row whose cell is blank or unparseable has no position relative to a computed - # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key` - # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]). - left_num = _lane_num(text) - if ref_value is None or left_num is None: - return False - return ((op == "gt" and left_num > ref_value) - or (op == "gte" and left_num >= ref_value) - or (op == "lt" and left_num < ref_value) - or (op == "lte" and left_num <= ref_value)) - wanted = str((condition or {}).get("value") or "").strip() - if op == "contains": - return wanted.casefold() in text.casefold() - if op == "not_contains": - return wanted.casefold() not in text.casefold() - if op in ("eq", "neq"): - left_num, right_num = _lane_num(text), _lane_num(wanted) - equal = (left_num == right_num if left_num is not None and right_num is not None - else text.casefold() == wanted.casefold()) - return equal if op == "eq" else not equal - ftype = (field_types or {}).get(field, "text") - left = _sort_key(text, ftype) - right = _sort_key(wanted, ftype) - if left is None or right is None: - return False - return ((op == "gt" and left > right) or (op == "gte" and left >= right) - or (op == "lt" and left < right) or (op == "lte" and left <= right)) - - -def compute_relation_cells(rt, table_key, tables=None): - """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. - - Zero store reads when the table declares neither kind — the same cheap-by-construction shape - `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. - """ - store = tables if tables is not None else ut_all(rt) - t = (store or {}).get(table_key) - fields = list((t or {}).get("fields") or []) - links = [f for f in fields if _ut().is_derived_link(f)] - rollups = [f for f in fields if isinstance(f.get("rollup"), dict)] - if not links and not rollups: - return 0 - rows = (t or {}).get("rows") or {} - by_key = {str(f.get("key")): f for f in fields} - - # --- resolve every link field ONCE per table, not once per row. - # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]` - resolved, linked_types = {}, {} - for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]: - lk_key = str((f or {}).get("key") or "") - if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict): - continue - bag = f["link"] - linked = (store or {}).get(str(bag.get("table") or "")) or {} - linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text") - for lf in (linked.get("fields") or [])} - if bag.get("inverse"): - # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary - # link cell contains this row id. The source cell remains the one relationship truth. - source_rows = linked.get("rows") or {} - inverse_key = str(bag.get("inverse") or "") - inverse_index = {} - for source_id, source_row in source_rows.items(): - for target_id in [part.strip() for part in - str((source_row or {}).get(inverse_key) or "").split(",")]: - if target_id: - inverse_index.setdefault(target_id, []).append( - (str(source_id), source_row or {})) - resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows} - elif bag.get("on"): - idx = _linked_rows_by_join(linked, str(bag["on"])) - from_key = _link_from_key(fields, bag) - resolved[lk_key] = { - str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), []) - for rid, row in rows.items()} if from_key else {} - else: - # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds - # are dropped rather than carried — a link to a deleted row is not a link. - lrows = (linked.get("rows") or {}) - resolved[lk_key] = { - str(rid): [(i, lrows[i]) for i in - [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] - if i and i in lrows] - for rid, row in rows.items()} - - changes = {} - for rid, row in rows.items(): - rid = str(rid) - row = row or {} - for f in links: - fk = str(f["key"]) - hits = (resolved.get(fk) or {}).get(rid) or [] - if f["link"].get("single"): - hits = hits[:1] - # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell - # is a projection of `resolved`, which is uncapped and is what every rollup below - # consumes — so a profile with 900 posts shows the first 500 ids and still averages - # over all 900. Same argument the `posts` window already makes: shedding is safe - # precisely because the authoritative store still holds everything. - want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) - if str(row.get(fk, "")) != want: - changes.setdefault(rid, {})[fk] = want - for f in rollups: - fk, bag = str(f["key"]), f["rollup"] - lk_key = str(bag.get("link") or "") - hits = list((resolved.get(lk_key) or {}).get(rid) or []) - # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING - # and therefore to a blank cell — never to a stale number. A column that keeps - # printing yesterday's answer after its input is gone is the worst of the options. - # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE - # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so - # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* - # rather than *the ones over X among the last 10* — two different windows wearing one - # sentence. Harmless while a threshold was a literal; incoherent the moment a - # threshold is a statistic OF the window, because the set being described and the set - # doing the describing would be different sets. - # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s - # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is - # computed over that window, THEN the conditions filter it") and the validator half - # was written against it. - # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH - # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` - # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` - # with no limit), so the blast radius is user-built rollups only. - # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last - # 12 posts, we also want to make it so its last N record, where the record's Status is - # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, - # so they select among the rows the window already kept. `where` selects WHICH rows - # the window is spent on. - # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per - # identity; run it first and a carousel could claim the slot its reel sibling needed, - # so the window would come up short for a reason nothing on screen explains. - # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this - # list, because at this point there is no fixed set for a statistic to be about. - where = list(bag.get("where") or []) - if where: - where_matches = lambda pair: [ - _rollup_condition_matches(pair[1], condition, - linked_types.get(lk_key) or {}, ref_value=None) - for condition in where] - if str(bag.get("whereConj") or "and") == "or": - hits = [pair for pair in hits if any(where_matches(pair))] - else: - hits = [pair for pair in hits if all(where_matches(pair))] - sort_by = str(bag.get("sortBy") or "") - if sort_by: - ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") - # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not - # rankable, and it must land at the END whichever direction is asked for — which - # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel - # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its - # window on rows that HAVE the value before it falls back to ones that do not. - keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] - rankable = [(p, k) for p, k in keyed if k is not None] - rankable.sort(key=lambda pk: pk[1], - reverse=str(bag.get("sortDir") or "desc") == "desc") - hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] - distinct_by = str(bag.get("distinctBy") or "") - if distinct_by: - seen, unique = set(), [] - for pair in hits: - identity = str(pair[1].get(distinct_by) or "").strip().lower() - # A blank is not an identity. Keep it rather than collapsing every unknown - # record into one synthetic duplicate. - if identity and identity in seen: - continue - if identity: - seen.add(identity) - unique.append(pair) - hits = unique - limit = int(bag.get("limit") or 0) - if limit: - hits = hits[:limit] - # --- the window is now FIXED, so a set-statistic threshold has a set to be about. - conditions = list(bag.get("conditions") or []) - if conditions: - # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the - # window; computing it inside `matches` would recompute the same mean for every - # candidate and — worse — would invite computing it over a set that the filter is - # already shrinking underneath it. - # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond - # the mean") is falsy, and the `or` spelling would silently rewrite it to the same - # number by accident. It reads identically and is right for the wrong reason, - # which is how it survives a review. - refs = [_rollup_ref_threshold( - [r for _i, r in hits], str(c.get("field") or ""), - (c.get("ref") or {}).get("sigmas", 0.0)) - if isinstance(c, dict) and c.get("ref") is not None else None - for c in conditions] - matches = lambda pair: [ - _rollup_condition_matches(pair[1], condition, - linked_types.get(lk_key) or {}, ref_value=ref) - for condition, ref in zip(conditions, refs)] - if str(bag.get("conditionConj") or "and") == "or": - hits = [pair for pair in hits if any(matches(pair))] - else: - hits = [pair for pair in hits if all(matches(pair))] - src = str(bag.get("field") or "") - # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the - # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at - # all, so the default stands for it. - want = _rollup_fold(str(bag.get("fn") or ""), - [r.get(src) for _i, r in hits] if src else [1] * len(hits), - (linked_types.get(lk_key) or {}).get(src, "text")) - if str(row.get(fk, "")) != want: - changes.setdefault(rid, {})[fk] = want - if not changes: - return 0 - - # A run that just wrote linked rows passes its in-flight user_tables bucket here so the - # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps - # the public helper's old persist-on-change behaviour. - if tables is not None: - tt = tables.get(table_key) - if tt is not None: - for r, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(r, {}).update(vals) - return len(changes) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for r, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(r, {}).update(vals) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return len(changes) - - -def _refresh_relations_inplace(tables, log=print): - """Refresh every relation against one mutable user_tables bucket; perform no store write.""" - touched = 0 - for tk, table in list((tables or {}).items()): - fields = (table or {}).get("fields") or [] - if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): - continue - try: - touched += compute_relation_cells(None, tk, tables=tables) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") - return touched - - + + +def _refresh_relations_inplace(tables, log=print, rt=None): + """Refresh every relation against one mutable user_tables bucket; perform no store write. + + ⭐ W41-T18 — `rt` RIDES THROUGH, and it used to be dropped on the floor. This path passed + `None`, which is fine for a `ut_*` link (the bucket is already in hand) and is the difference + between resolving and refusing for a link whose TARGET is `customer_data` or `product_data`: + a registry topic's pool is per tenant, so without the handle it cannot be built at all. + ⚠ Nothing else changes — `compute_relation_cells` reads `rt` only when `tables is None` + (`ut_all`) or when it persists, and both of those branches are unreachable from here. + ⛔ `rt` RIDES POSITIONALLY, in the slot that already existed. The `except Exception` below + swallows a `TypeError` as readily as a store outage, so ANY change to the shape of the call + on the next line disables the entire relational pass with nothing red — measured, on a + keyword this function briefly added (see `compute_relation_cells`). + """ + touched = 0 + for tk, table in list((tables or {}).items()): + fields = (table or {}).get("fields") or [] + if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): + continue + try: + touched += compute_relation_cells(rt, tk, tables=tables) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + def refresh_relations(rt, log=print, tables=None, persist=True): - """The tick half of the relational pass — the twin of `refresh_metrics`. - - ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale - when table B gains a row, and A has no way to know that happened. Cheap by construction — a - table declaring neither kind costs one dict scan. - """ + """The tick half of the relational pass — the twin of `refresh_metrics`. + + ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale + when table B gains a row, and A has no way to know that happened. Cheap by construction — a + table declaring neither kind costs one dict scan. + """ if tables is not None: - local = _refresh_relations_inplace(tables, log=log) + local = _refresh_relations_inplace(tables, log=log, rt=rt) if not local or not persist: return local actual = [0] def _up(cur): cur = cur if isinstance(cur, dict) else {} - actual[0] = _refresh_relations_inplace(cur, log=log) + actual[0] = _refresh_relations_inplace(cur, log=log, rt=rt) return cur rt.update(UT_STORE_KEY, _up, flush="async") return actual[0] snapshot = { - str(key): {**(table or {}), - "rows": {str(rid): dict(row or {}) - for rid, row in ((table or {}).get("rows") or {}).items()}} - for key, table in (ut_all(rt) or {}).items() - } - if not _refresh_relations_inplace(snapshot, log=log): - return 0 - actual = [0] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - actual[0] = _refresh_relations_inplace(cur, log=log) - return cur - - # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). - # - # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and - # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. - # A derived-cell recompute has no business forcing a commit on someone typing. - # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new - # row lives only in the store cache for 2-20s, and this call — on the SAME key - # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the - # result. `POST /rows` answered 201 and the row was gone. - # - # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it - # is what makes the other ~39 sync writers of this key safe. This line is not that fix and - # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both - # landed together on purpose: one is correctness, one is the hot path. - rt.update(UT_STORE_KEY, _up, flush="async") - return actual[0] - - + str(key): {**(table or {}), + "rows": {str(rid): dict(row or {}) + for rid, row in ((table or {}).get("rows") or {}).items()}} + for key, table in (ut_all(rt) or {}).items() + } + if not _refresh_relations_inplace(snapshot, log=log, rt=rt): + return 0 + actual = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + actual[0] = _refresh_relations_inplace(cur, log=log, rt=rt) + return cur + + # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). + # + # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and + # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. + # A derived-cell recompute has no business forcing a commit on someone typing. + # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new + # row lives only in the store cache for 2-20s, and this call — on the SAME key + # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the + # result. `POST /rows` answered 201 and the row was gone. + # + # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it + # is what makes the other ~39 sync writers of this key safe. This line is not that fix and + # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both + # landed together on purpose: one is correctness, one is the hot path. + rt.update(UT_STORE_KEY, _up, flush="async") + return actual[0] + + def refresh_metrics(rt, today=None, log=print, tables=None, persist=True): - """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window - metric can never go staler than one tick while a scheduler exists. Cheap by construction — - a table without metric fields costs a dict scan and nothing else.""" + """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window + metric can never go staler than one tick while a scheduler exists. Cheap by construction — + a table without metric fields costs a dict scan and nothing else.""" snapshot = tables if tables is not None else ut_all(rt) touched = 0 for tk, t in snapshot.items(): @@ -13319,1180 +13691,1180 @@ def refresh_metrics(rt, today=None, log=print, tables=None, persist=True): try: touched += compute_metric_cells(rt, tk, today=today, tables=snapshot, persist=persist) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") - return touched - - -def purge_subject(rt, handle): - """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the - tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half - non-optional: a purge that missed the pooled copy would not be erasure). Returns - `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is - now ABSENT, which is the one aggregate whose drill is emptiness.""" - subject = str(handle or "").strip().lstrip("@").lower() - if not subject: - return {} - counts = {} - tables = ut_all(rt) - post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} - codes = {str(r.get("shortcode") or "") for r in post_rows.values() - if str((r or {}).get("influencer_key") or "").strip().lower() == subject} - - keeps = { - "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") - or "").strip().lower() != subject, - "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") - or "").strip().lower() != subject, - "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, - DISCOVER_TABLE: lambda r: str((r or {}).get("handle") - or "").strip().lower() != subject, - } - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for tk, keep in keeps.items(): - t = cur.get(tk) - if t is None: - continue - rows = t.get("rows") or {} - nxt = {rid: r for rid, r in rows.items() if keep(r)} - counts[tk] = len(rows) - len(nxt) - t["rows"] = nxt - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - import ig_master - for bucket, n in (ig_master.purge_handle(subject) or {}).items(): - counts[f"master:{bucket}"] = n - return counts - - -# --------------------------------------------------------------------------------------------- -# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) -# --------------------------------------------------------------------------------------------- -# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | -# webhook | email. The first two are what always existed (Run now; cron via the tick). The four -# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: -# -# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, -# not for being in it. Implemented as per-record ARMED state over successive evaluations -# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms -# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing -# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the -# disarmed set with everything currently matching, so already-matching records do not fire -# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). -# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only -# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers -# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never -# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by -# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as -# a statusNote) catches whatever construction did not foresee. -# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds -# instead of running — a partial run entry names the count and the deliberate way through -# (Run now). C4's discovery guard is this rule's special case. -# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record -# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); -# event_field fires every transition. - -# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── -# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that -# renames the same events is a second vocabulary to learn for no gain. -# -# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". -# Renaming the key would have orphaned every stored trigger in production for a caption; the key -# is the contract with the store, the label is the contract with the reader, and they are allowed -# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the -# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 -# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. -# -# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show -# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a -# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop -# class wearing a friendlier face; here the two lists are separate on purpose and the refusal -# names the state rather than pretending the key is unknown. -# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── -# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the -# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — -# it is the event this flow starts from. Picking it sets the definition's kind to -# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. -# -# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the -# schedule (or on Run now), so it stays out of both lists below and its node switch flips the -# CRON — see `TRIGGER_SCHEDULE_KEYS`. -# -# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and -# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in -# the owner's own words and has not changed; what changed is which tuple it sits in, because -# `clean_trigger` refuses the planned list with a sentence and accepts this one. -TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", - "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", - "tiktok_profile_match") -#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS -#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses -#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper -#: section is not a section of one. -#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an -#: action that drives a browser are different builds; this row must not be read as progress on D-51. -TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") -TRIGGER_LABELS = { - "manual": "Manual", - "schedule": "At a scheduled time", - "event_field": "When a record matches conditions", - "record_updated": "When a record is updated", - "record_created": "When a record is created", - "enters_view": "When a record enters a view", - "webhook": "When a webhook is received", - "email": "When an email arrives", - "form_submitted": "When a form is submitted", - "ig_profile_match": "When an Instagram profile fits a criteria", - "button_clicked": "When a button is clicked", - "comment_added": "When a comment is added", - "web_page_changed": "When a website page changes", - "tiktok_profile_match": "When a TikTok profile fits a criteria", -} - -# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── -# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a -# distinction about where a trigger came FROM rather than about what a person is choosing. The -# question the picker actually asks is: does this fire on TIME, on your own DATA, or because -# something OUTSIDE said so. Three answers, and every trigger has exactly one. -# -# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map -# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the -# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` -# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation -# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is -# the surface. So: -# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule -# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because -# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into -# Database. -# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, -# so the fallback is provably dead code in production and the classification is still -# mandatory. The fallback catches the accident; the gate stops it shipping. -# ⭐ WAVE 34 · R19 — CONNECTOR SITS ABOVE DATABASE, DIRECTLY UNDER TIME. Owner, verbatim: -# *"In the Trigger picker, the Connector section sits directly above the Database section, just -# under the Time trigger types."* So the two orders below are SWAPPED against wave 24's, and -# nothing else moved: same keys, same labels, same fallback. -# ⛔ THIS IS THE WHOLE OF R19 AND IT IS DELIBERATELY NOT A CLIENT CHANGE. `steps.ts::groupTriggers` -# sorts by each option's `groupOrder` and by nothing else, so re-ordering an array on the client -# would look right in a fixture and be wrong in production the moment the server re-sorted. -# ⚠ `verify_steps.py` CANNOT WITNESS THIS EDIT: its C2 leg supplies its OWN `groupOrder` in a -# fixture and asserts the client honours it, which stays true whatever these numbers say. The -# check that binds R19 to this table lives in `verify_automation` beside the vocab section. -TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, - "connector": {"label": "Connector", "order": 2}, - "database": {"label": "Database", "order": 3}, - "other": {"label": "Other", "order": 99}} -#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to -#: prevent — it is the soft landing, not a category anybody should be adding triggers to. -TRIGGER_GROUP_FALLBACK = "other" -TRIGGER_GROUP_OF = { - "manual": "time", "schedule": "time", - "event_field": "database", "record_updated": "database", "record_created": "database", - "enters_view": "database", "form_submitted": "database", - "button_clicked": "database", "comment_added": "database", - "email": "connector", "webhook": "connector", "ig_profile_match": "connector", - "web_page_changed": "connector", "tiktok_profile_match": "connector", -} -#: The SUB-group inside "Connector" — which connected thing this trigger comes through. -#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two -#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a -#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key -#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it -#: must group by it and render `label`, never look it up. Said here because the miss would be -#: silent and partial, which is the worst shape. -#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and -#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) -TRIGGER_CONNECTOR = { - "email": {"key": "gmail", "label": "Gmail"}, - "webhook": {"key": "webhooks", "label": "Webhooks"}, - # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. - # The owner, verbatim and for the third wave running: *"I say this multiple times already the - # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under - # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see - # the option to choose either Instagram OR TikTok. That's it."* - # - # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: - # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH - # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are - # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening - # this picker is choosing the capability first and the platform second. `verify_automation` - # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's - # ruling is most of why this complaint survived two waves — so it is INVERTED in this same - # change, comment included. - # - # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded - # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and - # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. - "ig_profile_match": {"key": "scraper", "label": "Scraper"}, - "web_page_changed": {"key": "scraper", "label": "Scraper"}, - "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, -} -#: Triggers that watch a database and therefore need one named before they can fire. -TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", - "form_submitted") -#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. -#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, -#: so flipping its node must flip the schedule. -#: -#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read -#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, -#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For -#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a -#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns -#: about eight lines into its own comment. Derived from one named set now, so a trigger added to -#: `TRIGGER_KEYS` cannot silently join the wrong side of it. -#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, -#: and forgetting it is precisely the failure this constant's own note describes: it watches no -#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left -#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while -#: the schedule kept firing — a switch that lies. -TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") -#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. -#: -#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire -#: reason this constant exists rather than the tuple above being sent. That set answers "which -#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual -#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw -#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run -#: now": a control contradicting its own description. -#: -#: D-55's history is why it is DERIVED rather than listed: the client carried -#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact -#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). -#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. -TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} -#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so -#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. -TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") -MAX_WATCH_FIELDS = 12 -#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, -#: and so would a deployment that prefers immediacy over coalescing. -EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) -FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses -FIRE_WINDOW_SECONDS = 300 -FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on -EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation -EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest -CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota - -EMAIL_FIELDS = [ - field_def("email_id", "Email id"), field_def("email_from", "From"), - field_def("email_subject", "Subject"), field_def("email_date", "Date"), - field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), -] - - -def clean_trigger(raw, previous=None): - """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and - means what it always meant: manual + whatever `schedule` says. - - ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry - with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT - rather than refused — the picker writes `{key}` first and the table/field after, the - wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the - wire so the surface says "finish setting this up" instead of snapping back to Manual. It - cannot fire while incomplete (the hooks match on the table it does not name), which is the - fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a - condition on a field the trigger does not watch) are still refused with the sentence — - incomplete is a state, wrong is not. - """ - if raw in (None, "", {}): - return (dict(previous) if isinstance(previous, dict) and previous else None), None - if not isinstance(raw, dict): - return None, "the trigger must be an object" - prev = previous if isinstance(previous, dict) else {} - key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), - 30).strip() - if key in TRIGGER_PLANNED: - # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence - # says WHY rather than "unknown trigger", because the picker legitimately showed it. - return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet. " - f"it renders so you can see it is coming, and it cannot be saved") - if key not in TRIGGER_KEYS: - return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) - if key == "schedule": - # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` - # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored - # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. - return None, None - if key == "manual": - # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** - # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user - # can just press Run to make the full flow work."* - # - # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys - # together with one argument — *"storing a no-op trigger would be a second copy of that - # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has - # another home; **manual has none.** Nothing anywhere recorded "this automation is - # manual", so storing it is not a duplicate: it is the only record there has ever been. - # - # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so - # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the - # "nobody has decided yet" empty state, and Configuration — including the Database picker - # a plain automation cannot do without — never rendered. The owner reported it twice. The - # previous note reasoned that a manual option *"would bounce straight back to this state - # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion - # was that it should be STORED. - # - # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off - # (Run now always works, which is the whole of what it means) and a switch that governs - # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the - # cron stays reachable — picking Manual says how it fires today, never that it may not be - # scheduled tomorrow. - return {"key": "manual"}, None - out = {"key": key, - "enabled": bool(raw["enabled"]) if "enabled" in raw else - bool(prev.get("enabled", True)), - "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} - if key in TRIGGER_TABLE_KEYS: - table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, ("event triggers watch blank databases (ut_*) this wave. " - f"{table!r} is not one") - out["table"] = table - if key == "event_field": - # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record - # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a - # second, quieter way to express the same narrowing, and the owner asked for one. - # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is - # dropped on this definition's next clean — silently, and exactly once, because nothing - # writes the key back. A refusal here would have 400'd the live automations that carry it. - cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), - where="the trigger") - if cerr: - return None, cerr - out["when"] = cond - if key == "record_updated": - # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty - # is the WIDER reading and it is the default there too, so it stays the default here. - watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") - if watch_raw in (None, ""): - watch = [] - elif not isinstance(watch_raw, list): - return None, "the watched-field list must be a list of field keys" - else: - watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] - if len(watch) > MAX_WATCH_FIELDS: - return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " - f"fields. Leave the list empty to watch every field") - out["fields"] = watch - # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A - # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: - # "a record was updated" is an event, and asking it to also be a filter was the overlap - # with `event_field` the owner asked to end. Stated loudly in the contract AND here so - # nobody restores it as a bug fix. - # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` - # naturally returns "no gate" for it — the write itself becomes the event — rather than - # this needing a second removal anywhere. - if key == "enters_view": - out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), - 80).strip() - if key == "form_submitted": - # Blank = any form on that database. Naming one narrows to it, which is what a table - # carrying an intake form AND a correction form needs. - out["formToken"] = _s(raw.get("formToken") if "formToken" in raw - else prev.get("formToken"), 64).strip() - if key == "webhook": - # The token is MINTED here, once, and survives every later patch — rotating it on - # every Save would silently break the external caller the URL was given to. - out["token"] = _s(prev.get("token"), 64) or _secrets_token() - # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. - # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no - # map behaves exactly as it did — it fires the flow and reads nothing — so the live - # webhook automations are untouched. `webhook` deliberately stays OUT of - # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and - # every existing webhook trigger would go unconfigured and stop firing. - table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, ("a webhook writes into a blank database (ut_*). " - f"{table!r} is not one") - out["table"] = table - fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw - else prev.get("fieldMap")) - if ferr: - return None, ferr - out["fieldMap"] = fmap - if key == "email": - out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), - 200).strip() or "in:inbox is:unread" - out["configured"] = _trigger_configured(out) - return out, None - - -#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody -#: else's schema and is legitimately wider than an action's hand-written value list. -MAX_BODY_FIELDS = 40 -MAX_BODY_DEPTH = 5 - - -def clean_body_map(raw): - """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. - - Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated - rather than half-supported: `items.0.sku` would read as working for the first element and - silently write nothing the day a payload arrives with the list empty, which is the shape of - bug this module keeps paying for. A path that resolves to nothing writes nothing. - """ - if raw in (None, ""): - return {}, None - if not isinstance(raw, dict): - return None, "the webhook field map must be an object of {body path: field key}" - if len(raw) > MAX_BODY_FIELDS: - return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" - out = {} - for path, field in raw.items(): - p = _s(path, 200).strip() - if not p: - return None, "a webhook mapping has an empty body path" - if len(p.split(".")) > MAX_BODY_DEPTH: - return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload. " - f"map a shallower value") - fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") - if not fk: - return None, f"the value at {p!r} is not mapped to a field" - out[p] = fk[:60] - return out, None - - -def body_value(body, path): - """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is - an object or a list answers None rather than a stringified `{...}` in a cell, because the - Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" - cur = body - for part in str(path or "").split("."): - if not isinstance(cur, dict): - return None - cur = cur.get(part) - return cur if isinstance(cur, (str, int, float, bool)) else None - - -def webhook_row(rt, defn, body): - """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. - - ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same - `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, - from the only side the caller is on, from a payload whose paths did not resolve. That is the - D-11 class (a table that quietly stops growing), and the caller here is a machine that will - keep posting. The note rides the 200: the flow still fires, and the answer says why no row - was written. - - ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop - prevention law (A2(2)). A sibling automation watching this table does NOT fire on a - webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow - fires, because `hook_fire` fires it explicitly, which is the difference between "this - trigger fired" and "a write happened". - """ - trg = (defn or {}).get("trigger") or {} - table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) - if not table or not fmap or not isinstance(body, dict): - return "", 0, "" # no map configured — nothing to report - t = ut_get(rt, table) - if t is None: - return "", 0, f"{table} no longer exists, so nothing was written" - values = {} - for path, fkey in fmap.items(): - v = body_value(body, path) - if v is not None: - values[fkey] = _s(v, 500) if isinstance(v, str) else v - if not values: - return "", 0, ("none of the mapped paths resolved to a value in this payload. " - "check the paths against what you are sending") - rows = dict((t.get("rows") or {})) - if len(rows) >= row_cap(table): - return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " - f"written (the flow still ran)") - rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) - rows[rid] = values - ut_write_rows(rt, table, rows) - return rid, len(values), "" - - -def _trigger_configured(trg, config=None): - """Is this trigger complete enough to fire? One reader, because "configured" is asserted in - three places (the wire, the graph node, the hooks) and three copies of a boolean is how a - surface says "ready" about a trigger the engine skips. - - ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is - the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) - rather than on the trigger. `clean_trigger` calls this without it and so answers - conservatively (False); `clean_definition` calls it again with the validated config and - refines. Conservative-then-refined is the fail-closed order — the reverse would flash - "ready" on a trigger with nothing to search for. - """ - key = str((trg or {}).get("key") or "") - # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no - # filter is not a search, it is a request for the whole index. Named as a pair rather than - # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a - # boolean expression. - if key in ("ig_profile_match", "tiktok_profile_match"): - return bool((config or {}).get("predicates")) - if key in TRIGGER_TABLE_KEYS and not trg.get("table"): - return False - if key == "event_field": - # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, - # ever" — which is not a trigger, it is a description of the table. - # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone - # and carried no condition becomes `configured: false` on its next clean. It stops - # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it - # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. - return bool(trg.get("when")) - if key == "enters_view": - return bool(trg.get("viewId")) - return True - - -def _secrets_token(): - import secrets as _sec - return _sec.token_urlsafe(24) - - -# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the -# process would keep punishing an automation for a storm that ended with the restart. -_FIRES = {} -_FIRES_LOCK = threading.Lock() - - -def _breaker_trips(tenant, auto_id, now=None): - now = now if now is not None else time.time() - key = (tenant, str(auto_id)) - with _FIRES_LOCK: - log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] - log.append(now) - _FIRES[key] = log - return len(log) > FIRE_LIMIT - - -def _pause_trigger(rt, auto_id, note): - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = _s(note, 200) - return cur - _store_update(rt, _up, flush="sync") - - -def trigger_fire(rt, tenant, auto_id, log=print, rows=None): - """One trigger firing — breaker first, then the ordinary async run. False when it did not - start (breaker, or already running — both are answers, not errors).""" - if _breaker_trips(tenant, auto_id): - note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " - f"{FIRE_WINDOW_SECONDS // 60} minutes. Something is writing this trigger's " - f"subject in a loop") - _pause_trigger(rt, auto_id, note) - log(f"[aios-auto] breaker: {auto_id} {note}") - return False - return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) - - -# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE -# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). -_SETTLE = {} -_SETTLE_LOCK = threading.Lock() - - -def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): - """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. - - ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave - dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the - reason is worth stating because it is invisible from this file: an ordinary ut cell typed at - the grid door lands in the editor's PER-USER OVERLAY stratum - (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` - definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for - the common case the value that just changed exists ONLY in this event, and a definition-row - read sees the pre-write value — the trigger would evaluate stale and never fire. - `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree - read the record's other columns) with the written cells on top. - """ - key = (tenant, str(auto_id)) - cell = {str(field): after} if field else {} - if EVENT_SETTLE_SECONDS <= 0: - with _SETTLE_LOCK: - buf = _SETTLE.setdefault(key, {"rows": {}}) - buf["rows"].setdefault(str(row_id), {}).update(cell) - _settle_eval(rt, tenant, auto_id, log=log) - return - with _SETTLE_LOCK: - buf = _SETTLE.setdefault(key, {"rows": {}}) - buf["rows"].setdefault(str(row_id), {}).update(cell) - timer = buf.get("timer") - if timer is not None: - timer.cancel() # the burst continues — push the window out - timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, - args=(rt, tenant, auto_id), kwargs={"log": log}) - timer.daemon = True - buf["timer"] = timer - timer.start() - - -def view_filter(rt, table_key, view_id): - """`(tree, fields, problem)` for one saved view on a user table — the substrate the - `enters_view` trigger tests membership against (C3-v2 / owner R2). - - Personal strata first (`find_view`), then the shared bucket, because a view somebody shared - is exactly the kind an automation gets pointed at. A view that has been deleted answers a - PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one - would turn "when a record enters Overdue" into "on every write", which is the widening this - module refuses everywhere else. - """ - key = str(table_key or "") - vid = str(view_id or "").strip() - if not key or not vid: - return None, [], "the trigger names no view" - try: - import core.table_store as table_store - tops = table_store.make(f"{key}_table_workspace", st=rt) - found = tops.find_view(vid) - view = (found[1] if found else None) or tops.shared_view(vid) - except Exception as e: # noqa: BLE001 - return None, [], f"the view could not be read ({type(e).__name__})" - if not isinstance(view, dict): - return None, [], f"view {vid!r} no longer exists on {key}" - cfg = view.get("config") or {} - tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} - return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" - - -def _row_gate(rt, defn, trg): - """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. - - None means the trigger has NO match gate — the write itself is the event (wave 22's "the - field changed", and `record_updated` over any field). A problem means the gate cannot be - built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that - passes. - """ - key = trg.get("key") - if key == "enters_view": - tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) - if problem: - return None, problem - import harness.filter_eval as filter_eval - return (lambda row: filter_eval.matches(tree, row, fields)), "" - when = trg.get("when") - if when: - return (lambda row: lane_match(when, row)), "" - return None, "" - - -def _settle_eval(rt, tenant, auto_id, log=print): - """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" - with _SETTLE_LOCK: - buf = _SETTLE.pop((tenant, str(auto_id)), None) - written = dict((buf or {}).get("rows") or {}) - touched = list(written) - if not touched: - return - d = all_definitions(rt).get(str(auto_id)) - trg = (d or {}).get("trigger") or {} - if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ - or trg.get("paused") or not trg.get("enabled", True) \ - or not trg.get("configured", True): - return - gate, problem = _row_gate(rt, d, trg) - if problem: - # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken - # automation, not a quiet no-op — the note is what the surface shows instead of "On". - if (d.get("statusNote") or "") != problem: - _pause_trigger(rt, auto_id, problem) - log(f"[aios-auto] trigger gate: {auto_id} {problem}") - return - fired = [] - if gate is not None: - rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} - disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) - nxt = set(disarmed) - for rid in touched: - # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 - # tree can read the record's other columns, the just-written cells on top because - # for an ordinary ut column this event is the ONLY place that value exists yet. - if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), - **(written.get(rid) or {})}): - if rid not in disarmed: - fired.append(rid) # false→true THIS evaluation: the edge - nxt.add(rid) - else: - nxt.discard(rid) # left the state — re-armed (Airtable's rule) - if nxt != disarmed: - set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) - if not fired: - return - else: - fired = list(touched) # "the field changed" — the burst is the edge - if len(fired) > FLOOD_LIMIT: - _commit_run(rt, auto_id, "partial", - f"the trigger matched {len(fired)} records in one evaluation. More than " - f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " - f"process them deliberately", {"held": len(fired)}, True) - return - trigger_fire(rt, tenant, auto_id, log=log, rows=fired) - - -def _seed_event_state(rt, defn): - """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so - turning the trigger on fires nothing — the first fire needs a real false→true transition. - Evaluated over the definition rows (the machine-written truth this trigger class watches). - - Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation - reads. Two implementations of "does this row match" is how a seed disagrees with the edge it - is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. - """ - trg = (defn or {}).get("trigger") or {} - if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ - or not trg.get("configured"): - return - gate, problem = _row_gate(rt, defn, trg) - if gate is None or problem: - return # no match gate ⇒ nothing to arm; a broken gate seeds nothing - rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} - matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) - set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) - - -def grid_hook(evt): - """THE listener the human write doors emit into (registered onto - `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may - import both sides). Never raises into a write path; a broken trigger listener must not - break typing into a cell.""" - try: - st = evt.get("st") - if st is None: - return - tenant = str(getattr(st, "key", "") or "royal-imports") - table = str(evt.get("table") or "") - kind = str(evt.get("type") or "") - # ⭐⭐ WAVE 31 · T35 (D-134) — `cached=True`, and this is the ONLY caller that passes it. - # This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000 - # whole-document deep copies of the automations bucket, under the store lock, to re-read a - # trigger set that had not changed. See `all_definitions` for why the memo is safe here - # and nowhere else. - defs = all_definitions(st, cached=True) - field = str(evt.get("field") or "") - rid = str(evt.get("rowId") or "") - for aid, d in defs.items(): - trg = (d or {}).get("trigger") or {} - key = str(trg.get("key") or "") - if trg.get("paused") or not trg.get("enabled", True) \ - or not trg.get("configured", True) \ - or key not in TRIGGER_ROW_KEYS \ - or str(trg.get("table") or "") != table: - continue - if kind == "record_created" and key == "record_created": - hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 - ridn = int(rid) if rid.isdigit() else None - if ridn is None or ridn <= hw: - continue # once per record EVER (A2(4)) — undo-proof - set_state(st, aid, {"rcHighwater": ridn}) - # ⛔⛔ W31-T35 — MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT. - # This is the hazard a definitions memo creates and the reason D-134 is not a - # one-line change: this branch READS `rcHighwater` and WRITES it, so within one - # import burst the second row would compare against the highwater the FIRST row - # set — and read the pre-write value out of the memo, fire again, and break - # A2(4)'s *"once per record EVER — undo-proof"*. `set_state` goes through - # `_store_update`, which drops the memo, but `defs` is the object already in hand - # for the rest of THIS event; keeping the two in step is what makes the memo safe - # rather than merely fast [[read-path-cannot-witness-write-path]]. - (d.setdefault("state", {}))["rcHighwater"] = ridn - trigger_fire(st, tenant, aid, rows=[rid]) - elif kind == "event_field" and key == "event_field": - # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every - # write on the watched table settles, and the CONDITION decides whether it fires - # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would - # now always take its first arm anyway; leaving a read of a key the validator no - # longer writes is the drift seat this migration exists to close. - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - elif kind == "event_field" and key == "record_updated" and ( - not (trg.get("fields") or []) or field in (trg.get("fields") or [])): - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - elif key == "enters_view": - # BOTH kinds feed it: a row can enter a view by being edited into its filter or - # by being CREATED already inside it. Listening only to edits would silently miss - # every new record — the half of the definition a reader assumes is covered. - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - except Exception as e: # noqa: BLE001 - print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") - - -def form_fired(rt, table_key, row_id, values=None, form_token=""): - """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). - - One submitted form row → every `form_submitted` automation watching that database fires. - Returns the list of automation ids that started, so the door can log what it set off (and so - the gate can assert it, rather than asserting a side effect nobody can see). - - ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, - so it fires triggers exactly like typing into a cell does. The loop-prevention law is not - weakened by that — the engine's own writers still never reach here (only `routes_forms` calls - it), so an automation cannot create a form row and re-fire itself. - - `values` is accepted and unused today: the row is already written when this is called, and - the flow reads it from the table. It stays in the signature because the caller HAS it and a - later refire policy ("only when field X was submitted") needs it — a parameter added later - would mean changing D's call site in a wave that does not own it. - """ - started, table = [], str(table_key or "") - if not table: - return started - tenant = str(getattr(rt, "key", "") or "royal-imports") - for aid, d in all_definitions(rt).items(): - trg = (d or {}).get("trigger") or {} - if trg.get("key") != "form_submitted" or trg.get("paused") \ - or not trg.get("enabled", True) or not trg.get("configured", True): - continue - if str(trg.get("table") or "") != table: - continue - want = str(trg.get("formToken") or "") - if want and not hmac.compare_digest(want, str(form_token or "")): - continue # this automation watches a DIFFERENT form on that table - if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): - started.append(aid) - return started - - -def hook_fire(rt, tenant, auto_id, token, body=None): - """The webhook trigger's decision, separated from FastAPI so the gate can drive it. - Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, - 409 already running, 200 started. - - ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or - sent something that is not JSON. It is written onto a record BEFORE the flow fires — the - flow's actions walk the record the webhook just created, which is the whole point of mapping - it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. - """ - import hmac as _hmac - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return 404, {"error": "unknown_automation"} - trg = defn.get("trigger") or {} - want = str(trg.get("token") or "") - if trg.get("key") != "webhook" or not want: - return 403, {"error": "no_webhook", "message": - "this automation has no webhook trigger"} - if trg.get("paused") or not trg.get("enabled", True): - return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} - if not _hmac.compare_digest(want, str(token or "")): - return 403, {"error": "bad_token", "message": "that token is not valid"} - # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. - row_id, mapped, note = webhook_row(rt, defn, body) - started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) - out = {"started": bool(started), "at": _iso(), - # Answered even when zero, so a caller wiring a map up can see whether their paths - # resolved. Silence here would make "my JSON is not landing" undebuggable from the - # outside, which is the only side the caller is on. - "rowId": row_id or None, "mapped": mapped} - if note: - out["note"] = note # a 200 that wrote no row SAYS which reason it was - return 200, out - - -def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): - """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a - row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the - gate drives this without a network; production leaves them None. - - Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per - tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most - `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. - """ - trg = defn.get("trigger") or {} - if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): - return None - import oauth_connect - creator = str(defn.get("createdBy") or "").strip() or "admin" - token, err = oauth_connect.google_creds(rt, creator) - if err: - if (defn.get("statusNote") or "") != err: - def _note(cur): - cur = cur if isinstance(cur, dict) else {} - dd = cur.get(str(auto_id)) - if dd is not None: - dd["statusNote"] = _s(err, 200) - return cur - _store_update(rt, _note, flush="sync") - return None - lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) - reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) - ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) - if lerr: - return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer. {lerr}", - {}, True) - seen = set((defn.get("state") or {}).get("emailSeen") or []) - fresh = [m for m in ids if m not in seen] - if not fresh: - return None # nothing new is not a run — no history spam - if len(fresh) > FLOOD_LIMIT: - return _commit_run(rt, auto_id, "partial", - f"{len(fresh)} new emails matched in one poll. More than the " - f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " - f"the query, or press Run now after adjusting it", - {"held": len(fresh)}, True) - fresh = fresh[:EMAIL_MAX_PER_POLL] - rows_in, notes = [], [] - for mid in fresh: - row, rerr = reader(mid) - if row: - rows_in.append(row) - elif rerr: - notes.append(rerr) - table_key = (defn.get("config") or {}).get("targetTable") or "" - if not table_key: - return _commit_run(rt, auto_id, "error", - "the email trigger has nowhere to write. The automation names no " - "target database", {}, False) - ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", - EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) - ut_write_rows(rt, table_key, merged) - new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] - set_state(rt, auto_id, {"emailSeen": new_seen}) - summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched. " - f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") - if notes: - summary += f". {notes[0][:100]}" - entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, - True, affected=list(merged)[:200]) - trigger_fire(rt, tenant, auto_id, log=log) - return entry - - -def compose_sentence(defn): - """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the - definition so it cannot lie about what runs.""" - cfg = defn.get("config") or {} - trg = defn.get("trigger") or {} - sched = defn.get("schedule") or {} - kind = defn.get("kind") - if trg.get("key") == "event_field": - # Law 4: no watched field any more, so the sentence stops naming one. It said - # "When None changes on ut_x" the moment the key stopped being stored. - head = f"When a record in {trg.get('table')} matches conditions" - elif trg.get("key") == "record_updated": - head = f"When a record in {trg.get('table')} is updated" - elif trg.get("key") == "ig_profile_match": - head = "When an Instagram profile fits the criteria" - elif trg.get("key") == "tiktok_profile_match": - head = "When a TikTok profile fits the criteria" - elif trg.get("key") == "record_created": - head = f"When a record is created in {trg.get('table')}" - elif trg.get("key") == "webhook": - head = "When the webhook is called" - elif trg.get("key") == "email": - head = f"When an email matches {trg.get('query')}" - elif sched.get("enabled"): - head = f"{_cron_label(sched.get('cron'))}" - else: - head = "When you press Run now" - if kind == "scrape_db": - host = urlparse(str(cfg.get("url") or "")).hostname or "the page" - body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" - elif kind == "field_instagram": - # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts - # first" / ", anonymous only" described a choice that no longer exists. - body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" - elif kind in DISCOVERY_KINDS: - # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind - # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the - # `plain` branch below and introduced itself as *"do nothing yet — this automation has no - # actions"*: a flatly false sentence, on the one surface whose stated promise is that it - # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image - # incident — it USED to be this arm, and described every plain automation as an Instagram - # search for 0 profiles. The same two branches have now mis-described each other's - # automations in both directions, which is why neither may be a fallthrough.) - _dplatform, _dtable, _, _ = discovery_facts(kind) - body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " - f"{cfg.get('targetTable') or _dtable}") - else: - # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. - # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every - # plain automation would have introduced itself as "search Instagram for up to 0 profiles - # into ut_ig_candidates" — a sentence composed from a definition that says none of it, - # on the one surface whose whole promise is that it "cannot lie about what runs". - n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) - tbl = cfg.get("targetTable") or trg.get("table") or "" - body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) - if n else "do nothing yet. This automation has no actions") - lanes = cfg.get("lanes") or [] - tail = f", then route each record across {len(lanes)} lanes" if lanes else "" - return f"{head}, {body}{tail}." - - -def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): - """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it - directly. Returns the run entry, or None when it was already running (the 409).""" - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return None - # ⛔⛔ WAVE 32 · T45 (owner item 10) — AN UNCONFIGURED ACTION BLOCKS THE RUN, HERE, WHERE EVERY - # DOOR PASSES. The route checks too so a person gets a 400 rather than a silent no-op, but the - # tick and the webhook do not go through the route; a client-only block is not a block (D-112). - # ⚠ BEFORE `_claim`, deliberately: claiming and then refusing would leave the automation - # marked running until the release, i.e. a refusal that also produces a phantom 409 for the - # next honest attempt. - _refusal = run_refusal(defn) - if _refusal: - log(f"[aios-auto] refused: {_refusal}") - return None - if not _claim(tenant, auto_id): - return None - # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever - # happened to press Run — and above all not to the scheduler, which is not a person and - # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by - # whether a human or a cron got to the first run first. - owner = str(defn.get("createdBy") or "").strip() - if owner and username in MACHINE_OWNERS: - username = owner - try: - _step(tenant, auto_id, "running") - # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it - # first and do not start another profile scrape while it is outstanding: re-running the - # action would buy duplicate engagement reads and reintroduce the timeout this handoff - # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. - # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one - # rung earlier: a profile snapshot the vendor is still building is paid work, and - # starting a fresh scrape for the same handle would buy the identical row a second time. - # Ahead of the metric collector because the profile IS the thing the run was asked for; - # the engagement batches hang off it. - if _pending_profile_tasks(defn): - state, summary, counts, affected, steps = collect_pending_profile_snapshots( - rt, defn, username=username, log=log, - step=lambda text: _step(tenant, auto_id, text)) - return _commit_run(rt, auto_id, state, summary, - {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, - state != "error", affected, steps, - notes=counts.get(RUN_NOTES_KEY)) - if _pending_metric_tasks(defn): - state, summary, counts, affected, steps = collect_pending_metric_snapshots( - rt, defn, username=username, log=log, - step=lambda text: _step(tenant, auto_id, text)) - return _commit_run(rt, auto_id, state, summary, - {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, - state != "error", affected, steps, - notes=counts.get(RUN_NOTES_KEY)) - runner = RUNNERS.get(defn.get("kind")) - if runner is None: - return _commit_run(rt, auto_id, "error", - f"unknown automation kind {defn.get('kind')!r}", {}, False) - try: - # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. - # `status.step` was already on the wire and D's half renders it; measured against the - # code, it was set exactly ONCE ("running") and never again, so the word would have - # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a - # constant as a progress indicator is worse than rendering nothing: it looks like an - # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. - state, summary, counts, affected, steps = runner( - rt, defn, username=username, log=log, - step=lambda text: _step(tenant, auto_id, text), rows=rows) - except Refused as e: - return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") - return _commit_run(rt, auto_id, "error", - f"{type(e).__name__}: {str(e)[:200]}", {}, False) - # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is - # committed, over the records this run actually touched. ONE call site rather than three - # inside the runners: every kind gets actions and endings for free, and a fourth runner - # cannot forget to opt in. - # - # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, - # wired to the wire and covered by twelve gate checks that all called `apply_actions` - # DIRECTLY — so the whole feature was green and unreachable. A person would have built a - # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. - # - # A failing action must not fail the RUN: the machine steps already wrote their rows and - # reporting that as an error would misdescribe what happened. It degrades to `partial` - # with the reason in the summary — the cap_note discipline. - # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, - # which now reads this name — an assignment only on the success path would turn any - # action failure into a NameError inside the handler that exists to prevent exactly that. - # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked - # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the - # run a person stares at wondering why the automation stopped doing anything. - run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) - try: - a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], - username=username, log=log, - step=lambda text: _step(tenant, auto_id, text)) - # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so - # the runner contract keeps its shape, and they must leave before the merge or they - # would be a "count" everywhere downstream. - run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) - counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} - if a_counts.get("enrichMetricBatchesPending"): - batches = int(a_counts["enrichMetricBatchesPending"]) - state = "partial" if state != "error" else state - summary += (f". {batches} post-engagement batch" - f"{'' if batches == 1 else 'es'} still building; Views and other " - "metrics will be collected automatically without another paid read") - if a_counts.get("enrichUnbound"): - # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is - # `partial` because it genuinely did part of its job — it walked the records — and - # the sentence names the ONE thing that has to change, in the two places a person - # can change it. The old behaviour was `ok` with an empty table. - state = "partial" if state != "error" else state - summary += (". The Instagram step did not run: this database has no profile " - "column. Name one on the step, or mark a text column as the " - "Instagram profile" - + _unbound_hint(rt, _flow_table(defn))) - if a_counts.get("ttEnrichUnbound"): - # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a - # variable. A flow may carry BOTH steps, and the fix a person has to apply is - # per-column: naming an Instagram profile column does nothing for a TikTok step, - # so a single sentence covering "the enrich step" would send them to the wrong - # place half the time. Both may appear on one run, which is correct. - state = "partial" if state != "error" else state - summary += (". The TikTok step did not run: this database has no TikTok profile " - "column. Name one on the step, or mark a text column as a TikTok " - "profile" - + _unbound_hint(rt, _flow_table(defn))) - if a_counts.get("ttEnrichBlocked"): - state = "partial" if state != "error" else state - tt_note = next((n for n in run_notes if "(TikTok): " in n), "") - summary += (f". {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " - "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) - if a_counts.get("enrichProfileBatchesPending"): - # ⭐ The paid profile the vendor is still building. Said out loud so a run that - # looks like a failure is read as the handoff it is — the tick finishes it. - n = int(a_counts["enrichProfileBatchesPending"]) - state = "partial" if state != "error" else state - summary += (f". {n} profile read{'' if n == 1 else 's'} took longer than the " - "wait allows and will be collected automatically, at no extra cost") - if a_counts.get("enrichBlocked"): - state = "partial" if state != "error" else state - # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile - # read(s) were blocked" is the exact string the owner read three mornings running - # before asking "wtf is going on"; it names a quantity and withholds the one - # thing that would let anybody act. The first note is the vendor's own words. - # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same - # reason. Two sentences quoting each other's vendor reason is worse than one. - blocked_note = next((n for n in run_notes - if ": " in n and "(TikTok): " not in n), "") - summary += (f". {int(a_counts['enrichBlocked'])} profile read(s) were blocked" - + (f". {_s(blocked_note, 220)}" if blocked_note else "")) - # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the - # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it - # logged the cap and rolled up `ok`, so a flow that had silently stopped writing - # looked exactly like one that had nothing to write. Same rule, same sentence shape. - if a_counts.get("createCapped"): - state = "partial" if state != "error" else state - summary = (f"{summary}. {a_counts['createCapped']} row(s) NOT created: a target " - f"database is at its row cap") - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") - state = "partial" if state != "error" else state - summary = f"{summary}. The actions did not finish ({type(e).__name__})" - return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, - steps, notes=run_notes) - finally: - _release(tenant, auto_id) - - + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def purge_subject(rt, handle): + """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the + tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half + non-optional: a purge that missed the pooled copy would not be erasure). Returns + `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is + now ABSENT, which is the one aggregate whose drill is emptiness.""" + subject = str(handle or "").strip().lstrip("@").lower() + if not subject: + return {} + counts = {} + tables = ut_all(rt) + post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} + codes = {str(r.get("shortcode") or "") for r in post_rows.values() + if str((r or {}).get("influencer_key") or "").strip().lower() == subject} + + keeps = { + "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, + DISCOVER_TABLE: lambda r: str((r or {}).get("handle") + or "").strip().lower() != subject, + } + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for tk, keep in keeps.items(): + t = cur.get(tk) + if t is None: + continue + rows = t.get("rows") or {} + nxt = {rid: r for rid, r in rows.items() if keep(r)} + counts[tk] = len(rows) - len(nxt) + t["rows"] = nxt + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + import ig_master + for bucket, n in (ig_master.purge_handle(subject) or {}).items(): + counts[f"master:{bucket}"] = n + return counts + + +# --------------------------------------------------------------------------------------------- +# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) +# --------------------------------------------------------------------------------------------- +# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | +# webhook | email. The first two are what always existed (Run now; cron via the tick). The four +# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: +# +# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, +# not for being in it. Implemented as per-record ARMED state over successive evaluations +# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms +# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing +# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the +# disarmed set with everything currently matching, so already-matching records do not fire +# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). +# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only +# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers +# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never +# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by +# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as +# a statusNote) catches whatever construction did not foresee. +# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds +# instead of running — a partial run entry names the count and the deliberate way through +# (Run now). C4's discovery guard is this rule's special case. +# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record +# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); +# event_field fires every transition. + +# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── +# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that +# renames the same events is a second vocabulary to learn for no gain. +# +# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". +# Renaming the key would have orphaned every stored trigger in production for a caption; the key +# is the contract with the store, the label is the contract with the reader, and they are allowed +# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the +# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 +# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. +# +# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show +# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a +# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop +# class wearing a friendlier face; here the two lists are separate on purpose and the refusal +# names the state rather than pretending the key is unknown. +# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── +# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the +# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — +# it is the event this flow starts from. Picking it sets the definition's kind to +# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. +# +# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the +# schedule (or on Run now), so it stays out of both lists below and its node switch flips the +# CRON — see `TRIGGER_SCHEDULE_KEYS`. +# +# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and +# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in +# the owner's own words and has not changed; what changed is which tuple it sits in, because +# `clean_trigger` refuses the planned list with a sentence and accepts this one. +TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", + "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", + "tiktok_profile_match") +#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS +#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses +#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper +#: section is not a section of one. +#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an +#: action that drives a browser are different builds; this row must not be read as progress on D-51. +TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") +TRIGGER_LABELS = { + "manual": "Manual", + "schedule": "At a scheduled time", + "event_field": "When a record matches conditions", + "record_updated": "When a record is updated", + "record_created": "When a record is created", + "enters_view": "When a record enters a view", + "webhook": "When a webhook is received", + "email": "When an email arrives", + "form_submitted": "When a form is submitted", + "ig_profile_match": "When an Instagram profile fits a criteria", + "button_clicked": "When a button is clicked", + "comment_added": "When a comment is added", + "web_page_changed": "When a website page changes", + "tiktok_profile_match": "When a TikTok profile fits a criteria", +} + +# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── +# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a +# distinction about where a trigger came FROM rather than about what a person is choosing. The +# question the picker actually asks is: does this fire on TIME, on your own DATA, or because +# something OUTSIDE said so. Three answers, and every trigger has exactly one. +# +# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map +# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the +# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` +# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation +# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is +# the surface. So: +# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule +# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because +# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into +# Database. +# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, +# so the fallback is provably dead code in production and the classification is still +# mandatory. The fallback catches the accident; the gate stops it shipping. +# ⭐ WAVE 34 · R19 — CONNECTOR SITS ABOVE DATABASE, DIRECTLY UNDER TIME. Owner, verbatim: +# *"In the Trigger picker, the Connector section sits directly above the Database section, just +# under the Time trigger types."* So the two orders below are SWAPPED against wave 24's, and +# nothing else moved: same keys, same labels, same fallback. +# ⛔ THIS IS THE WHOLE OF R19 AND IT IS DELIBERATELY NOT A CLIENT CHANGE. `steps.ts::groupTriggers` +# sorts by each option's `groupOrder` and by nothing else, so re-ordering an array on the client +# would look right in a fixture and be wrong in production the moment the server re-sorted. +# ⚠ `verify_steps.py` CANNOT WITNESS THIS EDIT: its C2 leg supplies its OWN `groupOrder` in a +# fixture and asserts the client honours it, which stays true whatever these numbers say. The +# check that binds R19 to this table lives in `verify_automation` beside the vocab section. +TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, + "connector": {"label": "Connector", "order": 2}, + "database": {"label": "Database", "order": 3}, + "other": {"label": "Other", "order": 99}} +#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to +#: prevent — it is the soft landing, not a category anybody should be adding triggers to. +TRIGGER_GROUP_FALLBACK = "other" +TRIGGER_GROUP_OF = { + "manual": "time", "schedule": "time", + "event_field": "database", "record_updated": "database", "record_created": "database", + "enters_view": "database", "form_submitted": "database", + "button_clicked": "database", "comment_added": "database", + "email": "connector", "webhook": "connector", "ig_profile_match": "connector", + "web_page_changed": "connector", "tiktok_profile_match": "connector", +} +#: The SUB-group inside "Connector" — which connected thing this trigger comes through. +#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two +#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a +#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key +#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it +#: must group by it and render `label`, never look it up. Said here because the miss would be +#: silent and partial, which is the worst shape. +#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and +#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) +TRIGGER_CONNECTOR = { + "email": {"key": "gmail", "label": "Gmail"}, + "webhook": {"key": "webhooks", "label": "Webhooks"}, + # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. + # The owner, verbatim and for the third wave running: *"I say this multiple times already the + # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under + # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see + # the option to choose either Instagram OR TikTok. That's it."* + # + # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: + # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH + # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are + # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening + # this picker is choosing the capability first and the platform second. `verify_automation` + # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's + # ruling is most of why this complaint survived two waves — so it is INVERTED in this same + # change, comment included. + # + # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded + # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and + # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. + "ig_profile_match": {"key": "scraper", "label": "Scraper"}, + "web_page_changed": {"key": "scraper", "label": "Scraper"}, + "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, +} +#: Triggers that watch a database and therefore need one named before they can fire. +TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", + "form_submitted") +#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. +#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, +#: so flipping its node must flip the schedule. +#: +#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read +#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, +#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For +#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a +#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns +#: about eight lines into its own comment. Derived from one named set now, so a trigger added to +#: `TRIGGER_KEYS` cannot silently join the wrong side of it. +#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, +#: and forgetting it is precisely the failure this constant's own note describes: it watches no +#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left +#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while +#: the schedule kept firing — a switch that lies. +TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") +#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. +#: +#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire +#: reason this constant exists rather than the tuple above being sent. That set answers "which +#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual +#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw +#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run +#: now": a control contradicting its own description. +#: +#: D-55's history is why it is DERIVED rather than listed: the client carried +#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact +#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). +#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. +TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} +#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so +#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. +TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") +MAX_WATCH_FIELDS = 12 +#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, +#: and so would a deployment that prefers immediacy over coalescing. +EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) +FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses +FIRE_WINDOW_SECONDS = 300 +FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on +EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation +EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest +CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota + +EMAIL_FIELDS = [ + field_def("email_id", "Email id"), field_def("email_from", "From"), + field_def("email_subject", "Subject"), field_def("email_date", "Date"), + field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), +] + + +def clean_trigger(raw, previous=None): + """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and + means what it always meant: manual + whatever `schedule` says. + + ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry + with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT + rather than refused — the picker writes `{key}` first and the table/field after, the + wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the + wire so the surface says "finish setting this up" instead of snapping back to Manual. It + cannot fire while incomplete (the hooks match on the table it does not name), which is the + fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a + condition on a field the trigger does not watch) are still refused with the sentence — + incomplete is a state, wrong is not. + """ + if raw in (None, "", {}): + return (dict(previous) if isinstance(previous, dict) and previous else None), None + if not isinstance(raw, dict): + return None, "the trigger must be an object" + prev = previous if isinstance(previous, dict) else {} + key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), + 30).strip() + if key in TRIGGER_PLANNED: + # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence + # says WHY rather than "unknown trigger", because the picker legitimately showed it. + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet. " + f"it renders so you can see it is coming, and it cannot be saved") + if key not in TRIGGER_KEYS: + return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) + if key == "schedule": + # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` + # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored + # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. + return None, None + if key == "manual": + # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** + # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user + # can just press Run to make the full flow work."* + # + # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys + # together with one argument — *"storing a no-op trigger would be a second copy of that + # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has + # another home; **manual has none.** Nothing anywhere recorded "this automation is + # manual", so storing it is not a duplicate: it is the only record there has ever been. + # + # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so + # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the + # "nobody has decided yet" empty state, and Configuration — including the Database picker + # a plain automation cannot do without — never rendered. The owner reported it twice. The + # previous note reasoned that a manual option *"would bounce straight back to this state + # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion + # was that it should be STORED. + # + # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off + # (Run now always works, which is the whole of what it means) and a switch that governs + # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the + # cron stays reachable — picking Manual says how it fires today, never that it may not be + # scheduled tomorrow. + return {"key": "manual"}, None + out = {"key": key, + "enabled": bool(raw["enabled"]) if "enabled" in raw else + bool(prev.get("enabled", True)), + "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} + if key in TRIGGER_TABLE_KEYS: + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("event triggers watch blank databases (ut_*) this wave. " + f"{table!r} is not one") + out["table"] = table + if key == "event_field": + # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record + # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a + # second, quieter way to express the same narrowing, and the owner asked for one. + # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is + # dropped on this definition's next clean — silently, and exactly once, because nothing + # writes the key back. A refusal here would have 400'd the live automations that carry it. + cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), + where="the trigger") + if cerr: + return None, cerr + out["when"] = cond + if key == "record_updated": + # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty + # is the WIDER reading and it is the default there too, so it stays the default here. + watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") + if watch_raw in (None, ""): + watch = [] + elif not isinstance(watch_raw, list): + return None, "the watched-field list must be a list of field keys" + else: + watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] + if len(watch) > MAX_WATCH_FIELDS: + return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " + f"fields. Leave the list empty to watch every field") + out["fields"] = watch + # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A + # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: + # "a record was updated" is an event, and asking it to also be a filter was the overlap + # with `event_field` the owner asked to end. Stated loudly in the contract AND here so + # nobody restores it as a bug fix. + # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` + # naturally returns "no gate" for it — the write itself becomes the event — rather than + # this needing a second removal anywhere. + if key == "enters_view": + out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), + 80).strip() + if key == "form_submitted": + # Blank = any form on that database. Naming one narrows to it, which is what a table + # carrying an intake form AND a correction form needs. + out["formToken"] = _s(raw.get("formToken") if "formToken" in raw + else prev.get("formToken"), 64).strip() + if key == "webhook": + # The token is MINTED here, once, and survives every later patch — rotating it on + # every Save would silently break the external caller the URL was given to. + out["token"] = _s(prev.get("token"), 64) or _secrets_token() + # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. + # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no + # map behaves exactly as it did — it fires the flow and reads nothing — so the live + # webhook automations are untouched. `webhook` deliberately stays OUT of + # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and + # every existing webhook trigger would go unconfigured and stop firing. + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("a webhook writes into a blank database (ut_*). " + f"{table!r} is not one") + out["table"] = table + fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw + else prev.get("fieldMap")) + if ferr: + return None, ferr + out["fieldMap"] = fmap + if key == "email": + out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), + 200).strip() or "in:inbox is:unread" + out["configured"] = _trigger_configured(out) + return out, None + + +#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody +#: else's schema and is legitimately wider than an action's hand-written value list. +MAX_BODY_FIELDS = 40 +MAX_BODY_DEPTH = 5 + + +def clean_body_map(raw): + """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. + + Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated + rather than half-supported: `items.0.sku` would read as working for the first element and + silently write nothing the day a payload arrives with the list empty, which is the shape of + bug this module keeps paying for. A path that resolves to nothing writes nothing. + """ + if raw in (None, ""): + return {}, None + if not isinstance(raw, dict): + return None, "the webhook field map must be an object of {body path: field key}" + if len(raw) > MAX_BODY_FIELDS: + return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" + out = {} + for path, field in raw.items(): + p = _s(path, 200).strip() + if not p: + return None, "a webhook mapping has an empty body path" + if len(p.split(".")) > MAX_BODY_DEPTH: + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload. " + f"map a shallower value") + fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") + if not fk: + return None, f"the value at {p!r} is not mapped to a field" + out[p] = fk[:60] + return out, None + + +def body_value(body, path): + """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is + an object or a list answers None rather than a stringified `{...}` in a cell, because the + Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" + cur = body + for part in str(path or "").split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur if isinstance(cur, (str, int, float, bool)) else None + + +def webhook_row(rt, defn, body): + """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. + + ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same + `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, + from the only side the caller is on, from a payload whose paths did not resolve. That is the + D-11 class (a table that quietly stops growing), and the caller here is a machine that will + keep posting. The note rides the 200: the flow still fires, and the answer says why no row + was written. + + ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop + prevention law (A2(2)). A sibling automation watching this table does NOT fire on a + webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow + fires, because `hook_fire` fires it explicitly, which is the difference between "this + trigger fired" and "a write happened". + """ + trg = (defn or {}).get("trigger") or {} + table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) + if not table or not fmap or not isinstance(body, dict): + return "", 0, "" # no map configured — nothing to report + t = ut_get(rt, table) + if t is None: + return "", 0, f"{table} no longer exists, so nothing was written" + values = {} + for path, fkey in fmap.items(): + v = body_value(body, path) + if v is not None: + values[fkey] = _s(v, 500) if isinstance(v, str) else v + if not values: + return "", 0, ("none of the mapped paths resolved to a value in this payload. " + "check the paths against what you are sending") + rows = dict((t.get("rows") or {})) + if len(rows) >= row_cap(table): + return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " + f"written (the flow still ran)") + rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) + rows[rid] = values + ut_write_rows(rt, table, rows) + return rid, len(values), "" + + +def _trigger_configured(trg, config=None): + """Is this trigger complete enough to fire? One reader, because "configured" is asserted in + three places (the wire, the graph node, the hooks) and three copies of a boolean is how a + surface says "ready" about a trigger the engine skips. + + ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is + the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) + rather than on the trigger. `clean_trigger` calls this without it and so answers + conservatively (False); `clean_definition` calls it again with the validated config and + refines. Conservative-then-refined is the fail-closed order — the reverse would flash + "ready" on a trigger with nothing to search for. + """ + key = str((trg or {}).get("key") or "") + # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no + # filter is not a search, it is a request for the whole index. Named as a pair rather than + # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a + # boolean expression. + if key in ("ig_profile_match", "tiktok_profile_match"): + return bool((config or {}).get("predicates")) + if key in TRIGGER_TABLE_KEYS and not trg.get("table"): + return False + if key == "event_field": + # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, + # ever" — which is not a trigger, it is a description of the table. + # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone + # and carried no condition becomes `configured: false` on its next clean. It stops + # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it + # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. + return bool(trg.get("when")) + if key == "enters_view": + return bool(trg.get("viewId")) + return True + + +def _secrets_token(): + import secrets as _sec + return _sec.token_urlsafe(24) + + +# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the +# process would keep punishing an automation for a storm that ended with the restart. +_FIRES = {} +_FIRES_LOCK = threading.Lock() + + +def _breaker_trips(tenant, auto_id, now=None): + now = now if now is not None else time.time() + key = (tenant, str(auto_id)) + with _FIRES_LOCK: + log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] + log.append(now) + _FIRES[key] = log + return len(log) > FIRE_LIMIT + + +def _pause_trigger(rt, auto_id, note): + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = _s(note, 200) + return cur + _store_update(rt, _up, flush="sync") + + +def trigger_fire(rt, tenant, auto_id, log=print, rows=None): + """One trigger firing — breaker first, then the ordinary async run. False when it did not + start (breaker, or already running — both are answers, not errors).""" + if _breaker_trips(tenant, auto_id): + note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " + f"{FIRE_WINDOW_SECONDS // 60} minutes. Something is writing this trigger's " + f"subject in a loop") + _pause_trigger(rt, auto_id, note) + log(f"[aios-auto] breaker: {auto_id} {note}") + return False + return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) + + +# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE +# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). +_SETTLE = {} +_SETTLE_LOCK = threading.Lock() + + +def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): + """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. + + ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave + dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the + reason is worth stating because it is invisible from this file: an ordinary ut cell typed at + the grid door lands in the editor's PER-USER OVERLAY stratum + (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` + definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for + the common case the value that just changed exists ONLY in this event, and a definition-row + read sees the pre-write value — the trigger would evaluate stale and never fire. + `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree + read the record's other columns) with the written cells on top. + """ + key = (tenant, str(auto_id)) + cell = {str(field): after} if field else {} + if EVENT_SETTLE_SECONDS <= 0: + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + _settle_eval(rt, tenant, auto_id, log=log) + return + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + timer = buf.get("timer") + if timer is not None: + timer.cancel() # the burst continues — push the window out + timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, + args=(rt, tenant, auto_id), kwargs={"log": log}) + timer.daemon = True + buf["timer"] = timer + timer.start() + + +def view_filter(rt, table_key, view_id): + """`(tree, fields, problem)` for one saved view on a user table — the substrate the + `enters_view` trigger tests membership against (C3-v2 / owner R2). + + Personal strata first (`find_view`), then the shared bucket, because a view somebody shared + is exactly the kind an automation gets pointed at. A view that has been deleted answers a + PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one + would turn "when a record enters Overdue" into "on every write", which is the widening this + module refuses everywhere else. + """ + key = str(table_key or "") + vid = str(view_id or "").strip() + if not key or not vid: + return None, [], "the trigger names no view" + try: + import core.table_store as table_store + tops = table_store.make(f"{key}_table_workspace", st=rt) + found = tops.find_view(vid) + view = (found[1] if found else None) or tops.shared_view(vid) + except Exception as e: # noqa: BLE001 + return None, [], f"the view could not be read ({type(e).__name__})" + if not isinstance(view, dict): + return None, [], f"view {vid!r} no longer exists on {key}" + cfg = view.get("config") or {} + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" + + +def _row_gate(rt, defn, trg): + """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. + + None means the trigger has NO match gate — the write itself is the event (wave 22's "the + field changed", and `record_updated` over any field). A problem means the gate cannot be + built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that + passes. + """ + key = trg.get("key") + if key == "enters_view": + tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) + if problem: + return None, problem + import harness.filter_eval as filter_eval + return (lambda row: filter_eval.matches(tree, row, fields)), "" + when = trg.get("when") + if when: + return (lambda row: lane_match(when, row)), "" + return None, "" + + +def _settle_eval(rt, tenant, auto_id, log=print): + """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" + with _SETTLE_LOCK: + buf = _SETTLE.pop((tenant, str(auto_id)), None) + written = dict((buf or {}).get("rows") or {}) + touched = list(written) + if not touched: + return + d = all_definitions(rt).get(str(auto_id)) + trg = (d or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True): + return + gate, problem = _row_gate(rt, d, trg) + if problem: + # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken + # automation, not a quiet no-op — the note is what the surface shows instead of "On". + if (d.get("statusNote") or "") != problem: + _pause_trigger(rt, auto_id, problem) + log(f"[aios-auto] trigger gate: {auto_id} {problem}") + return + fired = [] + if gate is not None: + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) + nxt = set(disarmed) + for rid in touched: + # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 + # tree can read the record's other columns, the just-written cells on top because + # for an ordinary ut column this event is the ONLY place that value exists yet. + if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), + **(written.get(rid) or {})}): + if rid not in disarmed: + fired.append(rid) # false→true THIS evaluation: the edge + nxt.add(rid) + else: + nxt.discard(rid) # left the state — re-armed (Airtable's rule) + if nxt != disarmed: + set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) + if not fired: + return + else: + fired = list(touched) # "the field changed" — the burst is the edge + if len(fired) > FLOOD_LIMIT: + _commit_run(rt, auto_id, "partial", + f"the trigger matched {len(fired)} records in one evaluation. More than " + f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " + f"process them deliberately", {"held": len(fired)}, True) + return + trigger_fire(rt, tenant, auto_id, log=log, rows=fired) + + +def _seed_event_state(rt, defn): + """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so + turning the trigger on fires nothing — the first fire needs a real false→true transition. + Evaluated over the definition rows (the machine-written truth this trigger class watches). + + Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation + reads. Two implementations of "does this row match" is how a seed disagrees with the edge it + is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. + """ + trg = (defn or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or not trg.get("configured"): + return + gate, problem = _row_gate(rt, defn, trg) + if gate is None or problem: + return # no match gate ⇒ nothing to arm; a broken gate seeds nothing + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) + set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) + + +def grid_hook(evt): + """THE listener the human write doors emit into (registered onto + `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may + import both sides). Never raises into a write path; a broken trigger listener must not + break typing into a cell.""" + try: + st = evt.get("st") + if st is None: + return + tenant = str(getattr(st, "key", "") or "royal-imports") + table = str(evt.get("table") or "") + kind = str(evt.get("type") or "") + # ⭐⭐ WAVE 31 · T35 (D-134) — `cached=True`, and this is the ONLY caller that passes it. + # This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000 + # whole-document deep copies of the automations bucket, under the store lock, to re-read a + # trigger set that had not changed. See `all_definitions` for why the memo is safe here + # and nowhere else. + defs = all_definitions(st, cached=True) + field = str(evt.get("field") or "") + rid = str(evt.get("rowId") or "") + for aid, d in defs.items(): + trg = (d or {}).get("trigger") or {} + key = str(trg.get("key") or "") + if trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True) \ + or key not in TRIGGER_ROW_KEYS \ + or str(trg.get("table") or "") != table: + continue + if kind == "record_created" and key == "record_created": + hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 + ridn = int(rid) if rid.isdigit() else None + if ridn is None or ridn <= hw: + continue # once per record EVER (A2(4)) — undo-proof + set_state(st, aid, {"rcHighwater": ridn}) + # ⛔⛔ W31-T35 — MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT. + # This is the hazard a definitions memo creates and the reason D-134 is not a + # one-line change: this branch READS `rcHighwater` and WRITES it, so within one + # import burst the second row would compare against the highwater the FIRST row + # set — and read the pre-write value out of the memo, fire again, and break + # A2(4)'s *"once per record EVER — undo-proof"*. `set_state` goes through + # `_store_update`, which drops the memo, but `defs` is the object already in hand + # for the rest of THIS event; keeping the two in step is what makes the memo safe + # rather than merely fast [[read-path-cannot-witness-write-path]]. + (d.setdefault("state", {}))["rcHighwater"] = ridn + trigger_fire(st, tenant, aid, rows=[rid]) + elif kind == "event_field" and key == "event_field": + # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every + # write on the watched table settles, and the CONDITION decides whether it fires + # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would + # now always take its first arm anyway; leaving a read of a key the validator no + # longer writes is the drift seat this migration exists to close. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif kind == "event_field" and key == "record_updated" and ( + not (trg.get("fields") or []) or field in (trg.get("fields") or [])): + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif key == "enters_view": + # BOTH kinds feed it: a row can enter a view by being edited into its filter or + # by being CREATED already inside it. Listening only to edits would silently miss + # every new record — the half of the definition a reader assumes is covered. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") + + +def form_fired(rt, table_key, row_id, values=None, form_token=""): + """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). + + One submitted form row → every `form_submitted` automation watching that database fires. + Returns the list of automation ids that started, so the door can log what it set off (and so + the gate can assert it, rather than asserting a side effect nobody can see). + + ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, + so it fires triggers exactly like typing into a cell does. The loop-prevention law is not + weakened by that — the engine's own writers still never reach here (only `routes_forms` calls + it), so an automation cannot create a form row and re-fire itself. + + `values` is accepted and unused today: the row is already written when this is called, and + the flow reads it from the table. It stays in the signature because the caller HAS it and a + later refire policy ("only when field X was submitted") needs it — a parameter added later + would mean changing D's call site in a wave that does not own it. + """ + started, table = [], str(table_key or "") + if not table: + return started + tenant = str(getattr(rt, "key", "") or "royal-imports") + for aid, d in all_definitions(rt).items(): + trg = (d or {}).get("trigger") or {} + if trg.get("key") != "form_submitted" or trg.get("paused") \ + or not trg.get("enabled", True) or not trg.get("configured", True): + continue + if str(trg.get("table") or "") != table: + continue + want = str(trg.get("formToken") or "") + if want and not hmac.compare_digest(want, str(form_token or "")): + continue # this automation watches a DIFFERENT form on that table + if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): + started.append(aid) + return started + + +def hook_fire(rt, tenant, auto_id, token, body=None): + """The webhook trigger's decision, separated from FastAPI so the gate can drive it. + Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, + 409 already running, 200 started. + + ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or + sent something that is not JSON. It is written onto a record BEFORE the flow fires — the + flow's actions walk the record the webhook just created, which is the whole point of mapping + it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. + """ + import hmac as _hmac + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return 404, {"error": "unknown_automation"} + trg = defn.get("trigger") or {} + want = str(trg.get("token") or "") + if trg.get("key") != "webhook" or not want: + return 403, {"error": "no_webhook", "message": + "this automation has no webhook trigger"} + if trg.get("paused") or not trg.get("enabled", True): + return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} + if not _hmac.compare_digest(want, str(token or "")): + return 403, {"error": "bad_token", "message": "that token is not valid"} + # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. + row_id, mapped, note = webhook_row(rt, defn, body) + started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) + out = {"started": bool(started), "at": _iso(), + # Answered even when zero, so a caller wiring a map up can see whether their paths + # resolved. Silence here would make "my JSON is not landing" undebuggable from the + # outside, which is the only side the caller is on. + "rowId": row_id or None, "mapped": mapped} + if note: + out["note"] = note # a 200 that wrote no row SAYS which reason it was + return 200, out + + +def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): + """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a + row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the + gate drives this without a network; production leaves them None. + + Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per + tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most + `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. + """ + trg = defn.get("trigger") or {} + if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): + return None + import oauth_connect + creator = str(defn.get("createdBy") or "").strip() or "admin" + token, err = oauth_connect.google_creds(rt, creator) + if err: + if (defn.get("statusNote") or "") != err: + def _note(cur): + cur = cur if isinstance(cur, dict) else {} + dd = cur.get(str(auto_id)) + if dd is not None: + dd["statusNote"] = _s(err, 200) + return cur + _store_update(rt, _note, flush="sync") + return None + lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) + reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) + ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) + if lerr: + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer. {lerr}", + {}, True) + seen = set((defn.get("state") or {}).get("emailSeen") or []) + fresh = [m for m in ids if m not in seen] + if not fresh: + return None # nothing new is not a run — no history spam + if len(fresh) > FLOOD_LIMIT: + return _commit_run(rt, auto_id, "partial", + f"{len(fresh)} new emails matched in one poll. More than the " + f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " + f"the query, or press Run now after adjusting it", + {"held": len(fresh)}, True) + fresh = fresh[:EMAIL_MAX_PER_POLL] + rows_in, notes = [], [] + for mid in fresh: + row, rerr = reader(mid) + if row: + rows_in.append(row) + elif rerr: + notes.append(rerr) + table_key = (defn.get("config") or {}).get("targetTable") or "" + if not table_key: + return _commit_run(rt, auto_id, "error", + "the email trigger has nowhere to write. The automation names no " + "target database", {}, False) + ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", + EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) + ut_write_rows(rt, table_key, merged) + new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] + set_state(rt, auto_id, {"emailSeen": new_seen}) + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched. " + f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") + if notes: + summary += f". {notes[0][:100]}" + entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, + True, affected=list(merged)[:200]) + trigger_fire(rt, tenant, auto_id, log=log) + return entry + + +def compose_sentence(defn): + """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the + definition so it cannot lie about what runs.""" + cfg = defn.get("config") or {} + trg = defn.get("trigger") or {} + sched = defn.get("schedule") or {} + kind = defn.get("kind") + if trg.get("key") == "event_field": + # Law 4: no watched field any more, so the sentence stops naming one. It said + # "When None changes on ut_x" the moment the key stopped being stored. + head = f"When a record in {trg.get('table')} matches conditions" + elif trg.get("key") == "record_updated": + head = f"When a record in {trg.get('table')} is updated" + elif trg.get("key") == "ig_profile_match": + head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "tiktok_profile_match": + head = "When a TikTok profile fits the criteria" + elif trg.get("key") == "record_created": + head = f"When a record is created in {trg.get('table')}" + elif trg.get("key") == "webhook": + head = "When the webhook is called" + elif trg.get("key") == "email": + head = f"When an email matches {trg.get('query')}" + elif sched.get("enabled"): + head = f"{_cron_label(sched.get('cron'))}" + else: + head = "When you press Run now" + if kind == "scrape_db": + host = urlparse(str(cfg.get("url") or "")).hostname or "the page" + body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" + elif kind == "field_instagram": + # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts + # first" / ", anonymous only" described a choice that no longer exists. + body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" + elif kind in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind + # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the + # `plain` branch below and introduced itself as *"do nothing yet — this automation has no + # actions"*: a flatly false sentence, on the one surface whose stated promise is that it + # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image + # incident — it USED to be this arm, and described every plain automation as an Instagram + # search for 0 profiles. The same two branches have now mis-described each other's + # automations in both directions, which is why neither may be a fallthrough.) + _dplatform, _dtable, _, _ = discovery_facts(kind) + body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or _dtable}") + else: + # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. + # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every + # plain automation would have introduced itself as "search Instagram for up to 0 profiles + # into ut_ig_candidates" — a sentence composed from a definition that says none of it, + # on the one surface whose whole promise is that it "cannot lie about what runs". + n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) + tbl = cfg.get("targetTable") or trg.get("table") or "" + body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) + if n else "do nothing yet. This automation has no actions") + lanes = cfg.get("lanes") or [] + tail = f", then route each record across {len(lanes)} lanes" if lanes else "" + return f"{head}, {body}{tail}." + + +def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it + directly. Returns the run entry, or None when it was already running (the 409).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None + # ⛔⛔ WAVE 32 · T45 (owner item 10) — AN UNCONFIGURED ACTION BLOCKS THE RUN, HERE, WHERE EVERY + # DOOR PASSES. The route checks too so a person gets a 400 rather than a silent no-op, but the + # tick and the webhook do not go through the route; a client-only block is not a block (D-112). + # ⚠ BEFORE `_claim`, deliberately: claiming and then refusing would leave the automation + # marked running until the release, i.e. a refusal that also produces a phantom 409 for the + # next honest attempt. + _refusal = run_refusal(defn) + if _refusal: + log(f"[aios-auto] refused: {_refusal}") + return None + if not _claim(tenant, auto_id): + return None + # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever + # happened to press Run — and above all not to the scheduler, which is not a person and + # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by + # whether a human or a cron got to the first run first. + owner = str(defn.get("createdBy") or "").strip() + if owner and username in MACHINE_OWNERS: + username = owner + try: + _step(tenant, auto_id, "running") + # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it + # first and do not start another profile scrape while it is outstanding: re-running the + # action would buy duplicate engagement reads and reintroduce the timeout this handoff + # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. + # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one + # rung earlier: a profile snapshot the vendor is still building is paid work, and + # starting a fresh scrape for the same handle would buy the identical row a second time. + # Ahead of the metric collector because the profile IS the thing the run was asked for; + # the engagement batches hang off it. + if _pending_profile_tasks(defn): + state, summary, counts, affected, steps = collect_pending_profile_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + if _pending_metric_tasks(defn): + state, summary, counts, affected, steps = collect_pending_metric_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + runner = RUNNERS.get(defn.get("kind")) + if runner is None: + return _commit_run(rt, auto_id, "error", + f"unknown automation kind {defn.get('kind')!r}", {}, False) + try: + # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. + # `status.step` was already on the wire and D's half renders it; measured against the + # code, it was set exactly ONCE ("running") and never again, so the word would have + # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a + # constant as a progress indicator is worse than rendering nothing: it looks like an + # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. + state, summary, counts, affected, steps = runner( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text), rows=rows) + except Refused as e: + return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") + return _commit_run(rt, auto_id, "error", + f"{type(e).__name__}: {str(e)[:200]}", {}, False) + # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is + # committed, over the records this run actually touched. ONE call site rather than three + # inside the runners: every kind gets actions and endings for free, and a fourth runner + # cannot forget to opt in. + # + # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, + # wired to the wire and covered by twelve gate checks that all called `apply_actions` + # DIRECTLY — so the whole feature was green and unreachable. A person would have built a + # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. + # + # A failing action must not fail the RUN: the machine steps already wrote their rows and + # reporting that as an error would misdescribe what happened. It degrades to `partial` + # with the reason in the summary — the cap_note discipline. + # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, + # which now reads this name — an assignment only on the success path would turn any + # action failure into a NameError inside the handler that exists to prevent exactly that. + # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked + # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the + # run a person stares at wondering why the automation stopped doing anything. + run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) + try: + a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], + username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so + # the runner contract keeps its shape, and they must leave before the merge or they + # would be a "count" everywhere downstream. + run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) + counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} + if a_counts.get("enrichMetricBatchesPending"): + batches = int(a_counts["enrichMetricBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f". {batches} post-engagement batch" + f"{'' if batches == 1 else 'es'} still building; Views and other " + "metrics will be collected automatically without another paid read") + if a_counts.get("enrichUnbound"): + # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is + # `partial` because it genuinely did part of its job — it walked the records — and + # the sentence names the ONE thing that has to change, in the two places a person + # can change it. The old behaviour was `ok` with an empty table. + state = "partial" if state != "error" else state + summary += (". The Instagram step did not run: this database has no profile " + "column. Name one on the step, or mark a text column as the " + "Instagram profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichUnbound"): + # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a + # variable. A flow may carry BOTH steps, and the fix a person has to apply is + # per-column: naming an Instagram profile column does nothing for a TikTok step, + # so a single sentence covering "the enrich step" would send them to the wrong + # place half the time. Both may appear on one run, which is correct. + state = "partial" if state != "error" else state + summary += (". The TikTok step did not run: this database has no TikTok profile " + "column. Name one on the step, or mark a text column as a TikTok " + "profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichBlocked"): + state = "partial" if state != "error" else state + tt_note = next((n for n in run_notes if "(TikTok): " in n), "") + summary += (f". {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " + "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) + if a_counts.get("enrichProfileBatchesPending"): + # ⭐ The paid profile the vendor is still building. Said out loud so a run that + # looks like a failure is read as the handoff it is — the tick finishes it. + n = int(a_counts["enrichProfileBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f". {n} profile read{'' if n == 1 else 's'} took longer than the " + "wait allows and will be collected automatically, at no extra cost") + if a_counts.get("enrichBlocked"): + state = "partial" if state != "error" else state + # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile + # read(s) were blocked" is the exact string the owner read three mornings running + # before asking "wtf is going on"; it names a quantity and withholds the one + # thing that would let anybody act. The first note is the vendor's own words. + # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same + # reason. Two sentences quoting each other's vendor reason is worse than one. + blocked_note = next((n for n in run_notes + if ": " in n and "(TikTok): " not in n), "") + summary += (f". {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + + (f". {_s(blocked_note, 220)}" if blocked_note else "")) + # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the + # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it + # logged the cap and rolled up `ok`, so a flow that had silently stopped writing + # looked exactly like one that had nothing to write. Same rule, same sentence shape. + if a_counts.get("createCapped"): + state = "partial" if state != "error" else state + summary = (f"{summary}. {a_counts['createCapped']} row(s) NOT created: a target " + f"database is at its row cap") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") + state = "partial" if state != "error" else state + summary = f"{summary}. The actions did not finish ({type(e).__name__})" + return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, + steps, notes=run_notes) + finally: + _release(tenant, auto_id) + + def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): """Dispatch one run without keeping production work resident in the web process. @@ -14513,95 +14885,95 @@ def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): except ImportError: pass th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows), - daemon=True, name=f"automation-{tenant}-{auto_id}") - th.start() - return True - - -# --------------------------------------------------------------------------------------------- -# THE TICK + the in-process scheduler -# --------------------------------------------------------------------------------------------- - + daemon=True, name=f"automation-{tenant}-{auto_id}") + th.start() + return True + + +# --------------------------------------------------------------------------------------------- +# THE TICK + the in-process scheduler +# --------------------------------------------------------------------------------------------- + def pending_collect_ids(rt, definitions=None): - """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. - - ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking - forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and - stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is - true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), - so `due_ids` never returns it, nothing ever collected it, and the results the account had - already been charged for sat ready at the vendor forever. The sentence was not wrong; it was - describing a run nobody had scheduled. - - A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it - regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something - new*, this says *collect what is already running*, and folding the second into the first would - make an unscheduled automation's paid result depend on someone remembering to press a button. - """ - out = [] + """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. + + ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking + forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and + stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is + true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), + so `due_ids` never returns it, nothing ever collected it, and the results the account had + already been charged for sat ready at the vendor forever. The sentence was not wrong; it was + describing a run nobody had scheduled. + + A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it + regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something + new*, this says *collect what is already running*, and folding the second into the first would + make an unscheduled automation's paid result depend on someone remembering to press a button. + """ + out = [] definitions = all_definitions(rt) if definitions is None else definitions for aid, d in definitions.items(): - if not isinstance(d, dict): - continue - if (d.get("trigger") or {}).get("paused"): - continue - # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a - # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its - # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then - # never returned by this function — the tick collected nothing, forever. That is EXACTLY - # the live incident quoted in the docstring above (*"why is the SMALL 10 records test - # search taking forever, its not populating"*), reproduced for the second platform by the - # wave that added it: a result the tenant has already been charged for, stranded. - # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different - # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a - # kind string are not enumerable by memory, and this one is three thousand lines from the - # others. Pressing Run again does still collect (the runner's own branch reads the same - # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. - pending_discovery = (d.get("kind") in DISCOVERY_KINDS - and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) - # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile - # deferral would be stored and never collected, which is the same defect it fixes wearing - # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was - # measured on is `trigger: manual`. A capability written down but never walked is what - # this whole change is about, so it must not be reintroduced one function later. - if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): - out.append(aid) - return sorted(out) - - + if not isinstance(d, dict): + continue + if (d.get("trigger") or {}).get("paused"): + continue + # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a + # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its + # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then + # never returned by this function — the tick collected nothing, forever. That is EXACTLY + # the live incident quoted in the docstring above (*"why is the SMALL 10 records test + # search taking forever, its not populating"*), reproduced for the second platform by the + # wave that added it: a result the tenant has already been charged for, stranded. + # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different + # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a + # kind string are not enumerable by memory, and this one is three thousand lines from the + # others. Pressing Run again does still collect (the runner's own branch reads the same + # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. + pending_discovery = (d.get("kind") in DISCOVERY_KINDS + and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) + # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile + # deferral would be stored and never collected, which is the same defect it fixes wearing + # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was + # measured on is `trigger: manual`. A capability written down but never walked is what + # this whole change is about, so it must not be reintroduced one function later. + if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): + out.append(aid) + return sorted(out) + + def due_ids(rt, now=None, definitions=None): """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" definitions = all_definitions(rt) if definitions is None else definitions return sorted(aid for aid, d in definitions.items() if is_due(d, now)) - - + + def tick(rt, tenant, now=None, log=print): - """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the - ids started. The email polls are bounded and fail-quiet per automation — one broken - mailbox connection must not stop the tenant's schedules.""" + """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the + ids started. The email polls are bounded and fail-quiet per automation — one broken + mailbox connection must not stop the tenant's schedules.""" started = [] definitions = tick_definitions(rt) - # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the - # tenant has already been charged for; it is collected whether or not this automation is on a - # schedule. `_claim` makes the union safe — an id in both lists starts once. + # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the + # tenant has already been charged for; it is collected whether or not this automation is on a + # schedule. `_claim` makes the union safe — an id in both lists starts once. for aid in dict.fromkeys(list(pending_collect_ids(rt, definitions=definitions)) + list(due_ids(rt, now, definitions=definitions))): - if run_async(rt, tenant, aid, username="scheduler", log=log): - started.append(aid) + if run_async(rt, tenant, aid, username="scheduler", log=log): + started.append(aid) for aid, d in definitions.items(): - if (d.get("trigger") or {}).get("key") == "email": - try: - if email_poll(rt, tenant, aid, d, log=log) is not None: - started.append(aid) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") + if (d.get("trigger") or {}).get("key") == "email": + try: + if email_poll(rt, tenant, aid, d, log=log) is not None: + started.append(aid) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") # Derived cells are maintenance, not part of the scheduler's HTTP acknowledgement. The # background lane reads the large workspace at most once per UTC day, plus once after a real # automation run marks it dirty. An idle wake-up therefore stays rows-free, and Lambda never # times out waiting for a 40 MB tenant document to cross the public network. _start_derived_refresh(rt, tenant, now=now, log=log) - if started: - log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") + if started: + log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") return started @@ -14788,8 +15160,8 @@ def scheduler_status(): def tick_all(now=None, log=print): - """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop - the others' schedules.""" + """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop + the others' schedules.""" from harness import runtime as _rt out = {} slugs = _scheduler_tenants(_rt, now=now) @@ -14807,46 +15179,46 @@ def tick_all(now=None, log=print): _SCHEDULER_METRICS["lastTickAt"] = _iso() _SCHEDULER_METRICS["lastTenantCount"] = len(slugs) return out - - -#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer -#: would be a busy loop against a vocabulary that cannot express it. -TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) -_SCHEDULER = [None] - - -def scheduler_loop(log=print): - """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. - - Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and - `verify_seam` import this module through `main.py`, run against fake stores in seconds and - exit. A loop that ticked on entry would fire inside them. - """ - while True: - time.sleep(TICK_SECONDS) - try: - tick_all(log=log) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") - - + + +#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer +#: would be a busy loop against a vocabulary that cannot express it. +TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) +_SCHEDULER = [None] + + +def scheduler_loop(log=print): + """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. + + Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and + `verify_seam` import this module through `main.py`, run against fake stores in seconds and + exit. A loop that ticked on entry would fire inside them. + """ + while True: + time.sleep(TICK_SECONDS) + try: + tick_all(log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") + + def start_scheduler(log=print): - """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** - - ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave - brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 - seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires - two or three times inside a gate that imports `main.py` — and while a tick over a fake store - finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and - mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread - that can move a gate's subject is a flaky suite waiting to happen. - - So it takes the convention `main.py:353` already established for exactly this hazard — - `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup - event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the - external EventBridge tick (R5) does not depend on it either way, since that POSTs the - endpoint rather than riding this thread. - """ + """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** + + ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave + brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 + seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires + two or three times inside a gate that imports `main.py` — and while a tick over a fake store + finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and + mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread + that can move a gate's subject is a flaky suite waiting to happen. + + So it takes the convention `main.py:353` already established for exactly this hazard — + `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup + event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the + external EventBridge tick (R5) does not depend on it either way, since that POSTs the + endpoint rather than riding this thread. + """ if os.environ.get("AIOS_AUTOMATIONS") != "1": return False # PostgreSQL production has one durable wake-up: EventBridge -> Lambda -> protected endpoint. @@ -14856,9 +15228,9 @@ def start_scheduler(log=print): log("[aios-auto] in-process scheduler refused on PostgreSQL; use the external tick") return False if _SCHEDULER[0] is not None: - return False - th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, - daemon=True, name="automation-scheduler") - _SCHEDULER[0] = th - th.start() - return True + return False + th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, + daemon=True, name="automation-scheduler") + _SCHEDULER[0] = th + th.start() + return True diff --git a/api/connectors_tt.py b/api/connectors_tt.py index 60d96945a35f7fd8ca79443bf70cdec87e0a4c53..0603f4f23f7075252fdcef06ce19371ac1620523 100644 --- a/api/connectors_tt.py +++ b/api/connectors_tt.py @@ -1,568 +1,568 @@ -"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). - -Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an -automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file -exists at all rather than another thousand lines inside the engine. - -⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 -post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read -live from `GET /datasets/{id}/metadata` for **$0.00** and written down in -`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at -close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a -name below reads through a candidate list it is because the vendor has two names for one fact -(`biography`/`signature`, `region`/`country`), never because the name is uncertain. - -⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — -public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The -vendor key is a key to a SUPPLIER. - -⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE -OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and -`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not -Instagram's — and re-implementing them here would be a second copy of the deferral handling, the -truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. -Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from -`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. -⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because -the sentence above is the kind that quietly stops being true. - -⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** - 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared - envelope keys (`timestamp`, `input`) that no metadata call mentions; - 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` - and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one - TikTok claim that matters most (`play_count`) is exactly a declaration. -""" -from __future__ import annotations - -import automation_engine as engine -# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an -# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were -# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever -# read them off this module (the engine imports them from the transport itself), so they were four -# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. -from connectors_bd import ( - _bd_first_url, - _bd_flag, - _bd_list, - _bd_source_payload, - _first, - _ig_int, - bd_scrape, -) - -# --------------------------------------------------------------------------------------------- -# THE DATASETS -# --------------------------------------------------------------------------------------------- -# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET -# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a -# full field list, and the two DISCOVERY halves answer **404 for our key**: -# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). -# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's -# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. -TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records -TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields -TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields - -#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` -#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. -TT_POST_TYPE_VIDEO = "video" -TT_POST_TYPE_CONTENT = "content" - -#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept -#: beside the dataset ids because it is the same class of vendor fact. -#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical -#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own -#: batch calls, and the gate asserts the two agree rather than trusting that they do. -TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" - - -def tt_profile_url(handle): - """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" - h = str(handle or "").strip().lstrip("@") - return TT_PROFILE_URL.format(handle=h) if h else "" - - -# --------------------------------------------------------------------------------------------- -# THE FIELD MAPS -# --------------------------------------------------------------------------------------------- -# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared -# in `automation_engine`. The rules they all obey, stated once: -# -# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, -# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` -# returns `None` (never 0) when nothing matches, which is what makes that possible. -# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` -# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) -# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the -# vendor's side is preserved rather than silently discarded while our column model catches up. -# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` -# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they -# ride in `source_payload` where they make no claim. - - -def _tt_str(node, *names): - """The first non-empty string among `names`, or None when the vendor sent nothing. - - ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — - an empty string written into a cell claims "we looked and it is empty". - """ - v = _first(node, *names) - if v is None: - return None - s = str(v).strip() - return s or None - - -def _tt_pct(node, *names): - """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. - - ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to - the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — - measured on the Instagram side, and TikTok sends the same shape on all three of its rates. - """ - v = _first(node, *names) - if v is None: - return None - out = engine._pct100(v) - return out or None - - -def _tt_day(node, *names): - """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" - v = _first(node, *names) - if v is None: - return None - return engine._day(v) or None - - -def _drop_blanks(row): - """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" - return {k: v for k, v in row.items() if v is not None and v != ""} - - -def normalize_profile(node, handle=""): - """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. - - ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: - * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` - populated on 85% of rows and they carry the same text; - * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented - two-letter-ISO description, `country` has no description at all. - ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its - `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the - only count of its kind the dataset offers. - """ - node = node if isinstance(node, dict) else {} - account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "handle": account, - "full_name": _tt_str(node, "nickname"), - "tt_id": _tt_str(node, "id"), - "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), - "bio": _tt_str(node, "biography", "signature"), - "external_url": _bd_first_url(_first(node, "bio_link")), - "verified": _bd_flag(node, "is_verified"), - "is_private": _bd_flag(node, "is_private"), - # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* - # on their own description. It is the closest thing TikTok has to Instagram's - # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so - # the approximation only ever fills a cell the vendor actually answered. - "is_business": _bd_flag(node, "is_commerce_user"), - "followers": _ig_int(_first(node, "followers")), - "following": _ig_int(_first(node, "following")), - "posts_count": _ig_int(_first(node, "videos_count")), - # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no - # nulls). It is not a post-level number and it is not our `likes` column, which is why it - # is stored under a different name. - "likes_received": _ig_int(_first(node, "likes")), - "avg_engagement": _tt_pct(node, "awg_engagement_rate"), - "like_engagement": _tt_pct(node, "like_engagement_rate"), - "comment_engagement": _tt_pct(node, "comment_engagement_rate"), - "country_code": _tt_str(node, "region", "country"), - "region": _tt_str(node, "region"), - "predicted_lang": _tt_str(node, "predicted_lang"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT - # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, - # which is why the append law dates a snapshot by when WE read it. - "account_created_at": _tt_day(node, "create_time"), - "source_payload": _bd_source_payload(node), - }) - - -def tt_post_type(node): - """A TikTok post row → one of OUR three type options, or None. - - ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / - `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the - vendor's token would fail `_clean_field`'s option check on the way in and would put an - untranslated API word in front of a user on the way out. - - So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the - row carries more than one `carousel_images` entry, which is what a carousel IS on either - network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the - token cannot express it, so inferring `carousel` from the word would be inventing a fact. - ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and - that is exactly what would make the wrong default invisible). - """ - raw = str((node or {}).get("post_type") or "").strip().lower() - if raw == TT_POST_TYPE_VIDEO: - return "video" - if raw != TT_POST_TYPE_CONTENT: - return None - images = (node or {}).get("carousel_images") - return "carousel" if isinstance(images, list) and len(images) > 1 else "image" - - -def normalize_post(node): - """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. - - ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and - Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same - fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and - `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would - manufacture a second measurement that a rollup could average or double-count, which is a worse - outcome than the missing column it would paper over. - - ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and - the probe measured them as the same shape. That equality is what lets the comments→posts link - join with NO normaliser, which the Instagram side never had. - ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). - ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and - countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares - no equivalent, so `paid_partnership`/`partner` have no column on this family at all. - """ - node = node if isinstance(node, dict) else {} - shortcode = _tt_str(node, "shortcode", "post_id") - if not shortcode: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "shortcode": shortcode, - # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. - # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the - # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so - # one name means one thing across both corpora. Reading the display name silently broke the - # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched - # NOTHING, so a person could not filter posts by creator and a rollup would count zero. - # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a - # different fact, and filling a join key with it is worse than leaving it blank — a blank - # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. - # The URL is the honest second source: it carries the handle by construction. - "influencer_key": (_tt_str(node, "account_id") - or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), - "posted_at": _tt_day(node, "create_time"), - "type": tt_post_type(node), - "caption": _tt_str(node, "description"), - "url": _tt_str(node, "url"), - "hashtags": _bd_list(node, "hashtags"), - "tagged_location": _tt_str(node, "commerce_info"), - "views": _ig_int(_first(node, "play_count")), - "likes": _ig_int(_first(node, "digg_count")), - "comments": _ig_int(_first(node, "comment_count")), - "shares": _ig_int(_first(node, "num_share_count")), - "saves": _ig_int(_first(node, "collect_count")), - "video_duration": _ig_int(_first(node, "video_duration")), - "source_payload": _bd_source_payload(node), - }) - - -def normalize_comment(node): - """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. - - ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR - `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in - `source_payload`. Reading the array's length instead would be a second, disagreeing answer to - a question the vendor already answers — and it would disagree, because a page of replies is not - all of them. - ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is - text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor - that changes its mind cannot change ours. - ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged - PII) stay in `source_payload` and are promoted to no column, which is the same posture the - Instagram comment schema takes for the same D-24 reason. - """ - node = node if isinstance(node, dict) else {} - comment_key = _tt_str(node, "comment_id") - if not comment_key: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "comment_key": comment_key, - "shortcode": _tt_str(node, "post_id"), - # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the - # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); - # primary first, so a row carrying the rich form is not silently served the plain one. - # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. - "text": _tt_str(node, "comment_text", "comment_text_only"), - "commented_at": _tt_day(node, "date_created"), - "likes": _ig_int(_first(node, "num_likes")), - "replies": _ig_int(_first(node, "num_replies")), - "source_payload": _bd_source_payload(node), - }) - - -#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them -#: rather than naming three functions, and so adding a fourth dataset is one entry. -TT_NORMALIZERS = { - "tt_profile": normalize_profile, - "tt_post_metrics": normalize_post, - "tt_comments": normalize_comment, -} - - -# --------------------------------------------------------------------------------------------- -# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) -# --------------------------------------------------------------------------------------------- - -def tt_handle(url): - """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. - - Deliberately permissive about the input and strict about the output, because the two callers - hand it different things: an automation stores whatever a person typed in the profile column - (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean - `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot - resolve to two different handles. - """ - s = str(url or "").strip() - if not s: - return "" - if "tiktok.com" in s.lower(): - # Everything after the first `@`, up to the next path segment or query. - tail = s.split("@", 1)[1] if "@" in s else "" - s = tail.split("/")[0].split("?")[0].split("#")[0] - s = s.strip().lstrip("@").strip() - # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. - return s if s and all(c.isalnum() or c in "._" for c in s) else "" - - -def tt_post_urls(node, limit=0): - """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. - - ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row - we have already bought, so the posts read is a scrape of links we hold, never a search for them. - The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, - so a design that reached for either would not merely be dearer, it would not work. - - ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an - array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe - read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's - `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** - (measured below), so the reader built against the quoted sentence found nothing, forever, in - silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration - of X"* are different claims, and prose cannot be told apart by a reader downstream — which is - why the correction names the method, not just the value. - - ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, - and preferring whichever happened to be longer is how one creator's window silently differs - from another's. - ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, - validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is - never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, - not a number this function may invent. - """ - raw = (node or {}).get("top_videos") - out = [] - for item in raw if isinstance(raw, list) else []: - # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. - # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a - # live handle returns **19 DICTS**, keyed - # `video_url · video_id · playcount · diggcount · commentcount · share_count · - # favorites_count · create_date · cover_image`. - # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and - # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says - # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field - # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's - # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was - # the first. [[reachable-is-not-the-same-as-built]] - # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY - # list on every real profile — post capture could not have worked for anybody, and the - # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture - # written from a schema tests the schema. - # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept - # because it costs nothing and is what a future corpus revision would most likely use. The - # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents - # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as - # before, rather than to a URL built out of a guess. - # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring - # whichever array happened to be longer is how one creator's window silently differs from - # another's, and that reasoning is unchanged by this correction. - if isinstance(item, dict): - u = str(item.get("video_url") or item.get("url") or "").strip() - else: - u = str(item or "").strip() - if u.lower().startswith("http") and u not in out: - out.append(u) - return out[:limit] if limit and limit > 0 else out - - -def pull_posts_tt(post_urls, log=print, deferred=None): - """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. - - ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side - measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still - running at 67 minutes. Nothing here loops per URL. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] posts: {note}") - return [], note - out = [r for r in (normalize_post(n) for n in rows) if r] - return out, "" - - -def pull_comments_tt(post_urls, log=print, deferred=None): - """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. - - ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on - both networks: a comments scrape ingests identifiable third parties who never entered anybody's - list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes - none of them to a column; this function adds no new exposure, it just has to be asked for. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] comments: {note}") - return [], note - out = [r for r in (normalize_comment(n) for n in rows) if r] - return out, "" - - -#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. -#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral -#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's -#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse -#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. -TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) - - -def _media_deferrals(deferred): - """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. - - ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the - handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one - kind here, so the id already carries everything a mapper choice depends on — and importing - Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). - """ - out = [] - for d in deferred or []: - if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: - out.append(dict(d)) - return out - - -def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, - max_posts=0, post_metrics=False, comment_metrics=False): - """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. - - `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, - so the engine's enrich branch treats every network identically and no caller learns a new - shape. - - ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's - do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — - no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` - then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction - rather than by a rule: their input IS a post permalink, so asking for comments with post capture - off is a request with no subject, and it returns none instead of quietly buying posts nobody - asked for. - - ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than - pessimistic. The Instagram contract reads `ok` only when identity AND media both landed - (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower - count and no posts must not paint green over a posts table that did not grow"*). So: posts not - ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → - `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", - never "did the function finish". - - ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the - paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, - with its own note explaining that a multi-provider chain is a promise something walks it and - that nothing walks Instagram's second name today either. So a refusal here is final, and it - says so instead of implying a retry somewhere. - """ - handle = tt_handle(url) - if not handle: - return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", - "note": f"{url!r} is not a TikTok profile URL or handle"} - - # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from - # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not - # happen; a miss falls through to the single-URL call below. - cached = prefetch.get(handle) if isinstance(prefetch, dict) else None - _deferred = [] - if isinstance(cached, dict) and cached: - rows, note = [cached], "" - else: - rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) - - node = rows[0] if rows else {} - profile = normalize_profile(node, handle) if node else {} - # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` - # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and - # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly - # this reason, and answering "0 followers" instead is the failure it exists to prevent. - unreadable = profile.get("followers") is None and profile.get("following") is None - if note or unreadable: - # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still - # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the - # same record. That was live on the Instagram profile path until 2026-08-09 — measured on - # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being - # reintroduced here by omission. - if isinstance(pending_profile, list): - for d in _deferred: - pending_profile.append({**d, "kind": "profile", "influencer": handle}) - why = note or ("the scrape answered, but no follower/following counts were readable in it " - "(the field names may have moved - see tiktok-capture.md)") - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "deferredProfile": [d.get("snapshotId") for d in _deferred], - "note": why} - - # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ - if not post_metrics: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; post capture is off for this step"} - urls = tt_post_urls(node, limit=max_posts) - if not urls: - # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and - # saying so is what stops the next run paying to be told the same thing. - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; this account's row carried no post links"} - posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) - comments, c_note = ([], "") - if comment_metrics and posts: - # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — - # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused - # is not silently asked about again one rung later. - comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], - log=log, deferred=_deferred) - # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is - # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to - # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE - # deferral is impossible — the profile branch above returns `blocked` on any note — so **every - # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose - # collector writes preset profile cells onto somebody's record from post rows. The engine keeps - # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). - # ⇒ So this returns them under their OWN key, filtered by dataset identity - # (`_media_deferrals`), and the engine files them in the metric queue with the handle it - # already holds. Returning rather than appending also keeps the queue's vocabulary out of a - # connector: this module knows which CORPUS deferred, never what the engine calls it. - # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that - # matters most — that is exactly the run where the vendor took too long, so a caller reading - # the ids only from the success path would lose every batch it actually paid for. - deferred_media = _media_deferrals(_deferred) - if not posts: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", "deferredMedia": deferred_media, - "note": p_note or note or "profile read; the post source returned nothing"} - return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, - "via": "brightdata", "deferredMedia": deferred_media, - "note": c_note or note or ""} +"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). + +Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an +automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file +exists at all rather than another thousand lines inside the engine. + +⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 +post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read +live from `GET /datasets/{id}/metadata` for **$0.00** and written down in +`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a +name below reads through a candidate list it is because the vendor has two names for one fact +(`biography`/`signature`, `region`/`country`), never because the name is uncertain. + +⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — +public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The +vendor key is a key to a SUPPLIER. + +⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE +OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and +`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not +Instagram's — and re-implementing them here would be a second copy of the deferral handling, the +truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. +Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from +`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. +⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because +the sentence above is the kind that quietly stops being true. + +⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** + 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared + envelope keys (`timestamp`, `input`) that no metadata call mentions; + 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` + and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one + TikTok claim that matters most (`play_count`) is exactly a declaration. +""" +from __future__ import annotations + +import automation_engine as engine +# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an +# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were +# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever +# read them off this module (the engine imports them from the transport itself), so they were four +# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. +from connectors_bd import ( + _bd_first_url, + _bd_flag, + _bd_list, + _bd_source_payload, + _first, + _ig_int, + bd_scrape, +) + +# --------------------------------------------------------------------------------------------- +# THE DATASETS +# --------------------------------------------------------------------------------------------- +# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET +# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a +# full field list, and the two DISCOVERY halves answer **404 for our key**: +# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). +# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's +# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. +TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records +TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields +TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields + +#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` +#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. +TT_POST_TYPE_VIDEO = "video" +TT_POST_TYPE_CONTENT = "content" + +#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept +#: beside the dataset ids because it is the same class of vendor fact. +#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical +#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own +#: batch calls, and the gate asserts the two agree rather than trusting that they do. +TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" + + +def tt_profile_url(handle): + """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" + h = str(handle or "").strip().lstrip("@") + return TT_PROFILE_URL.format(handle=h) if h else "" + + +# --------------------------------------------------------------------------------------------- +# THE FIELD MAPS +# --------------------------------------------------------------------------------------------- +# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared +# in `automation_engine`. The rules they all obey, stated once: +# +# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, +# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` +# returns `None` (never 0) when nothing matches, which is what makes that possible. +# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` +# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) +# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the +# vendor's side is preserved rather than silently discarded while our column model catches up. +# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` +# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they +# ride in `source_payload` where they make no claim. + + +def _tt_str(node, *names): + """The first non-empty string among `names`, or None when the vendor sent nothing. + + ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — + an empty string written into a cell claims "we looked and it is empty". + """ + v = _first(node, *names) + if v is None: + return None + s = str(v).strip() + return s or None + + +def _tt_pct(node, *names): + """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. + + ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to + the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — + measured on the Instagram side, and TikTok sends the same shape on all three of its rates. + """ + v = _first(node, *names) + if v is None: + return None + out = engine._pct100(v) + return out or None + + +def _tt_day(node, *names): + """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" + v = _first(node, *names) + if v is None: + return None + return engine._day(v) or None + + +def _drop_blanks(row): + """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" + return {k: v for k, v in row.items() if v is not None and v != ""} + + +def normalize_profile(node, handle=""): + """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. + + ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: + * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` + populated on 85% of rows and they carry the same text; + * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented + two-letter-ISO description, `country` has no description at all. + ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its + `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the + only count of its kind the dataset offers. + """ + node = node if isinstance(node, dict) else {} + account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "handle": account, + "full_name": _tt_str(node, "nickname"), + "tt_id": _tt_str(node, "id"), + "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), + "bio": _tt_str(node, "biography", "signature"), + "external_url": _bd_first_url(_first(node, "bio_link")), + "verified": _bd_flag(node, "is_verified"), + "is_private": _bd_flag(node, "is_private"), + # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* + # on their own description. It is the closest thing TikTok has to Instagram's + # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so + # the approximation only ever fills a cell the vendor actually answered. + "is_business": _bd_flag(node, "is_commerce_user"), + "followers": _ig_int(_first(node, "followers")), + "following": _ig_int(_first(node, "following")), + "posts_count": _ig_int(_first(node, "videos_count")), + # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no + # nulls). It is not a post-level number and it is not our `likes` column, which is why it + # is stored under a different name. + "likes_received": _ig_int(_first(node, "likes")), + "avg_engagement": _tt_pct(node, "awg_engagement_rate"), + "like_engagement": _tt_pct(node, "like_engagement_rate"), + "comment_engagement": _tt_pct(node, "comment_engagement_rate"), + "country_code": _tt_str(node, "region", "country"), + "region": _tt_str(node, "region"), + "predicted_lang": _tt_str(node, "predicted_lang"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT + # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, + # which is why the append law dates a snapshot by when WE read it. + "account_created_at": _tt_day(node, "create_time"), + "source_payload": _bd_source_payload(node), + }) + + +def tt_post_type(node): + """A TikTok post row → one of OUR three type options, or None. + + ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / + `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the + vendor's token would fail `_clean_field`'s option check on the way in and would put an + untranslated API word in front of a user on the way out. + + So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the + row carries more than one `carousel_images` entry, which is what a carousel IS on either + network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the + token cannot express it, so inferring `carousel` from the word would be inventing a fact. + ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and + that is exactly what would make the wrong default invisible). + """ + raw = str((node or {}).get("post_type") or "").strip().lower() + if raw == TT_POST_TYPE_VIDEO: + return "video" + if raw != TT_POST_TYPE_CONTENT: + return None + images = (node or {}).get("carousel_images") + return "carousel" if isinstance(images, list) and len(images) > 1 else "image" + + +def normalize_post(node): + """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. + + ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and + Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same + fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and + `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would + manufacture a second measurement that a rollup could average or double-count, which is a worse + outcome than the missing column it would paper over. + + ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and + the probe measured them as the same shape. That equality is what lets the comments→posts link + join with NO normaliser, which the Instagram side never had. + ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). + ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and + countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares + no equivalent, so `paid_partnership`/`partner` have no column on this family at all. + """ + node = node if isinstance(node, dict) else {} + shortcode = _tt_str(node, "shortcode", "post_id") + if not shortcode: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "shortcode": shortcode, + # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. + # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the + # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so + # one name means one thing across both corpora. Reading the display name silently broke the + # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched + # NOTHING, so a person could not filter posts by creator and a rollup would count zero. + # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a + # different fact, and filling a join key with it is worse than leaving it blank — a blank + # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. + # The URL is the honest second source: it carries the handle by construction. + "influencer_key": (_tt_str(node, "account_id") + or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), + "posted_at": _tt_day(node, "create_time"), + "type": tt_post_type(node), + "caption": _tt_str(node, "description"), + "url": _tt_str(node, "url"), + "hashtags": _bd_list(node, "hashtags"), + "tagged_location": _tt_str(node, "commerce_info"), + "views": _ig_int(_first(node, "play_count")), + "likes": _ig_int(_first(node, "digg_count")), + "comments": _ig_int(_first(node, "comment_count")), + "shares": _ig_int(_first(node, "num_share_count")), + "saves": _ig_int(_first(node, "collect_count")), + "video_duration": _ig_int(_first(node, "video_duration")), + "source_payload": _bd_source_payload(node), + }) + + +def normalize_comment(node): + """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. + + ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR + `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in + `source_payload`. Reading the array's length instead would be a second, disagreeing answer to + a question the vendor already answers — and it would disagree, because a page of replies is not + all of them. + ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is + text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor + that changes its mind cannot change ours. + ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged + PII) stay in `source_payload` and are promoted to no column, which is the same posture the + Instagram comment schema takes for the same D-24 reason. + """ + node = node if isinstance(node, dict) else {} + comment_key = _tt_str(node, "comment_id") + if not comment_key: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "comment_key": comment_key, + "shortcode": _tt_str(node, "post_id"), + # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the + # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); + # primary first, so a row carrying the rich form is not silently served the plain one. + # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. + "text": _tt_str(node, "comment_text", "comment_text_only"), + "commented_at": _tt_day(node, "date_created"), + "likes": _ig_int(_first(node, "num_likes")), + "replies": _ig_int(_first(node, "num_replies")), + "source_payload": _bd_source_payload(node), + }) + + +#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them +#: rather than naming three functions, and so adding a fourth dataset is one entry. +TT_NORMALIZERS = { + "tt_profile": normalize_profile, + "tt_post_metrics": normalize_post, + "tt_comments": normalize_comment, +} + + +# --------------------------------------------------------------------------------------------- +# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) +# --------------------------------------------------------------------------------------------- + +def tt_handle(url): + """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. + + Deliberately permissive about the input and strict about the output, because the two callers + hand it different things: an automation stores whatever a person typed in the profile column + (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean + `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot + resolve to two different handles. + """ + s = str(url or "").strip() + if not s: + return "" + if "tiktok.com" in s.lower(): + # Everything after the first `@`, up to the next path segment or query. + tail = s.split("@", 1)[1] if "@" in s else "" + s = tail.split("/")[0].split("?")[0].split("#")[0] + s = s.strip().lstrip("@").strip() + # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. + return s if s and all(c.isalnum() or c in "._" for c in s) else "" + + +def tt_post_urls(node, limit=0): + """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. + + ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row + we have already bought, so the posts read is a scrape of links we hold, never a search for them. + The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, + so a design that reached for either would not merely be dearer, it would not work. + + ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an + array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe + read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's + `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** + (measured below), so the reader built against the quoted sentence found nothing, forever, in + silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration + of X"* are different claims, and prose cannot be told apart by a reader downstream — which is + why the correction names the method, not just the value. + + ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, + and preferring whichever happened to be longer is how one creator's window silently differs + from another's. + ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, + validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is + never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, + not a number this function may invent. + """ + raw = (node or {}).get("top_videos") + out = [] + for item in raw if isinstance(raw, list) else []: + # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. + # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a + # live handle returns **19 DICTS**, keyed + # `video_url · video_id · playcount · diggcount · commentcount · share_count · + # favorites_count · create_date · cover_image`. + # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and + # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says + # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field + # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's + # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was + # the first. [[reachable-is-not-the-same-as-built]] + # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY + # list on every real profile — post capture could not have worked for anybody, and the + # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture + # written from a schema tests the schema. + # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept + # because it costs nothing and is what a future corpus revision would most likely use. The + # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents + # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as + # before, rather than to a URL built out of a guess. + # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring + # whichever array happened to be longer is how one creator's window silently differs from + # another's, and that reasoning is unchanged by this correction. + if isinstance(item, dict): + u = str(item.get("video_url") or item.get("url") or "").strip() + else: + u = str(item or "").strip() + if u.lower().startswith("http") and u not in out: + out.append(u) + return out[:limit] if limit and limit > 0 else out + + +def pull_posts_tt(post_urls, log=print, deferred=None): + """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. + + ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side + measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still + running at 67 minutes. Nothing here loops per URL. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] posts: {note}") + return [], note + out = [r for r in (normalize_post(n) for n in rows) if r] + return out, "" + + +def pull_comments_tt(post_urls, log=print, deferred=None): + """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. + + ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on + both networks: a comments scrape ingests identifiable third parties who never entered anybody's + list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes + none of them to a column; this function adds no new exposure, it just has to be asked for. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] comments: {note}") + return [], note + out = [r for r in (normalize_comment(n) for n in rows) if r] + return out, "" + + +#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. +#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral +#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's +#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse +#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. +TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) + + +def _media_deferrals(deferred): + """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. + + ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the + handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one + kind here, so the id already carries everything a mapper choice depends on — and importing + Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). + """ + out = [] + for d in deferred or []: + if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: + out.append(dict(d)) + return out + + +def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, + max_posts=0, post_metrics=False, comment_metrics=False): + """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. + + `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, + so the engine's enrich branch treats every network identically and no caller learns a new + shape. + + ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's + do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — + no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` + then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction + rather than by a rule: their input IS a post permalink, so asking for comments with post capture + off is a request with no subject, and it returns none instead of quietly buying posts nobody + asked for. + + ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than + pessimistic. The Instagram contract reads `ok` only when identity AND media both landed + (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower + count and no posts must not paint green over a posts table that did not grow"*). So: posts not + ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → + `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", + never "did the function finish". + + ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the + paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, + with its own note explaining that a multi-provider chain is a promise something walks it and + that nothing walks Instagram's second name today either. So a refusal here is final, and it + says so instead of implying a retry somewhere. + """ + handle = tt_handle(url) + if not handle: + return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", + "note": f"{url!r} is not a TikTok profile URL or handle"} + + # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from + # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not + # happen; a miss falls through to the single-URL call below. + cached = prefetch.get(handle) if isinstance(prefetch, dict) else None + _deferred = [] + if isinstance(cached, dict) and cached: + rows, note = [cached], "" + else: + rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) + + node = rows[0] if rows else {} + profile = normalize_profile(node, handle) if node else {} + # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` + # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and + # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly + # this reason, and answering "0 followers" instead is the failure it exists to prevent. + unreadable = profile.get("followers") is None and profile.get("following") is None + if note or unreadable: + # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still + # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the + # same record. That was live on the Instagram profile path until 2026-08-09 — measured on + # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being + # reintroduced here by omission. + if isinstance(pending_profile, list): + for d in _deferred: + pending_profile.append({**d, "kind": "profile", "influencer": handle}) + why = note or ("the scrape answered, but no follower/following counts were readable in it " + "(the field names may have moved - see tiktok-capture.md)") + return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", + "deferredProfile": [d.get("snapshotId") for d in _deferred], + "note": why} + + # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ + if not post_metrics: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; post capture is off for this step"} + urls = tt_post_urls(node, limit=max_posts) + if not urls: + # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and + # saying so is what stops the next run paying to be told the same thing. + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; this account's row carried no post links"} + posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) + comments, c_note = ([], "") + if comment_metrics and posts: + # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — + # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused + # is not silently asked about again one rung later. + comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], + log=log, deferred=_deferred) + # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is + # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to + # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE + # deferral is impossible — the profile branch above returns `blocked` on any note — so **every + # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose + # collector writes preset profile cells onto somebody's record from post rows. The engine keeps + # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). + # ⇒ So this returns them under their OWN key, filtered by dataset identity + # (`_media_deferrals`), and the engine files them in the metric queue with the handle it + # already holds. Returning rather than appending also keeps the queue's vocabulary out of a + # connector: this module knows which CORPUS deferred, never what the engine calls it. + # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that + # matters most — that is exactly the run where the vendor took too long, so a caller reading + # the ids only from the success path would lose every batch it actually paid for. + deferred_media = _media_deferrals(_deferred) + if not posts: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", "deferredMedia": deferred_media, + "note": p_note or note or "profile read; the post source returned nothing"} + return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, + "via": "brightdata", "deferredMedia": deferred_media, + "note": c_note or note or ""} diff --git a/api/main.py b/api/main.py index e338505a3afbe136739ed1510e929deb61b53245..dd0b4eb668b9f3896772db912466a81120d0d35f 100644 --- a/api/main.py +++ b/api/main.py @@ -1,675 +1,675 @@ -"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. - -It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, -reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the -JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. - -WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: - - * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared - `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same - anonymous principal and no route could scope anything. Now a signed stateless cookie carries - a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the - user RECORD. `/api/health` stays unauthenticated (liveness only). - * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home - for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). - * **The SSL shim is env-gated.** It used to run at import, unconditionally. - -⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the -session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This -is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB -commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. -""" -import os -import sys -from pathlib import Path - -# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- -# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an -# unverified default context; the HF Space and the container are unaffected. This used to run -# unconditionally at import — i.e. the shipped container disabled TLS verification for every -# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a -# man-in-the-middle away from being someone else's data. It is now opt-in, must be set -# deliberately, and is never on by default. -# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. -if os.environ.get("AIOS_INSECURE_SSL") == "1": - import ssl - ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 - -# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container -# (where the layout differs from the local sibling-dir default). --- -_HERE = Path(__file__).resolve() -_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) -for _p in (str(_RI), str(_HERE.parent)): - if _p not in sys.path: - sys.path.insert(0, _p) - -from dotenv import load_dotenv # noqa: E402 -load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed - -from fastapi import Body, Depends, FastAPI, Request # noqa: E402 -from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 -from fastapi.responses import JSONResponse # noqa: E402 -from fastapi.staticfiles import StaticFiles # noqa: E402 -from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 - -import aios_session # noqa: E402 -import routes_admin # noqa: E402 -import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) -import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) -import routes_auth # noqa: E402 -import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) -import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) -import routes_customers # noqa: E402 -import routes_grid # noqa: E402 -import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) -import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) -import routes_nav # noqa: E402 -import routes_pages # noqa: E402 -import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) -import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) -import routes_records # noqa: E402 -import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) -import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) -import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) -import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) -import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) -import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) -import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) -import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) -import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) -import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) -import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) -import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) -import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) -import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) -import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) -import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) -import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) -import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) -import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router) -from core import grid_events # noqa: E402 -# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. -# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a -# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not. -from core import data_binding # noqa: E402 -from core import store # noqa: E402 -from deps import Session, module_gate # noqa: E402 - -_WEB_DIST = _HERE.parents[1] / "web" / "dist" - -# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- -# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never -# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the -# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so -# embed == standalone by construction. -# -# TWO REASONS THIS IS HERE and not folded into the routes: -# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR -# layout is broken, and discovering that from a 500 on the first customer request instead of -# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a -# rewrite dropped it and this restores it.) -# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and -# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this -# API have not drifted. The rewrite removed them and the referee went red; retargeting the -# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do -# not delete — but only when the subject genuinely moved. Here it should not have moved). -# -# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from -# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and -# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone -# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the -# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. -import json as _json_contract # noqa: E402 -_FIELDS_PATH = _RI / "aios_grid_fields.json" -if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " - "platform root (it also carries the data layer this API imports)." - ) -_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) -FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract -# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. -# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be -# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. -_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS - if f["source"] == "odoo" and f["type"] in ("text", "status", "select", - "date")} - -app = FastAPI(title="AIOS web API") - -# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with -# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the -# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. -app.add_middleware(GZipMiddleware, minimum_size=1024) - - -@app.exception_handler(StarletteHTTPException) -def _error_shape(request: Request, exc: StarletteHTTPException): - """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. - - `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a - 422 from a malformed path param) is wrapped here so a client never has to branch on two - different error bodies. - """ - detail = exc.detail - if isinstance(detail, dict) and "error" in detail: - body = detail - else: - body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} - return JSONResponse(body, status_code=exc.status_code, - headers=getattr(exc, "headers", None)) - - -@app.exception_handler(grid_events.StoreUnavailable) -def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): - """THE SAFETY NET for "the store is down" → 503, from anywhere. - - ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a - `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own - try/except — surfaced as a 500. A store outage is a normal operational state and every route - here touches the store at least twice (the workspace read, then the write), so "remember to - wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler - means a store outage can only ever be a 503, whichever call raised it. - - The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this - adapter, which has no durable session dict to degrade into. A 200 over a write that - evaporated is the failure the whole rule exists to prevent. - """ - return JSONResponse( - {"error": {"code": "store_unavailable", - "message": "the tenant store is unavailable. No change was saved"}}, - status_code=503) - - -@app.exception_handler(data_binding.StoreWriteRefused) -def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused): - """D-315 — this deployment may not write that store. 503, with the REASON. - - ⛔ IT ECHOES THE EXCEPTION'S OWN MESSAGE, WHERE ITS NEIGHBOUR ABOVE USES A FIXED SENTENCE, and - the difference is the point. "The store is unavailable" is the whole truth about an outage; a - refusal has a cause the operator cannot otherwise discover, because per D-160 the Space - environment cannot be read from outside. A refusal that said only "unavailable" would send - somebody hunting for a network fault that does not exist ([[report-the-cause-before-you-fix-it]]). - The message names the deployment, the store and the fix, and carries no customer data. - - ⛔ AND IT IS ITS OWN HANDLER RATHER THAN A SUBCLASS OF `StoreUnavailable`. Several routes carry - `except StoreUnavailable:` blocks that DEGRADE to a session-scoped fallback workspace — correct - for an outage, catastrophic here, because the user would be told the change was saved. A - distinct type falls through every one of them to this handler. - """ - return JSONResponse( - {"error": {"code": "store_write_refused", - "message": f"this deployment may not write the tenant store. No change was " - f"saved. {exc}"}}, - status_code=503) - - -@app.exception_handler(store.StoreConflict) -def _store_conflict(request: Request, exc: store.StoreConflict): - """D-305 — the document moved under this write more times than it could be rebased. 503. - - ⚠ 503 RATHER THAN 409, DELIBERATELY. A 409 invites the client to resolve a conflict, and there - is nothing here for it to resolve: the store already re-read and re-applied this change up to - `_MAX_REBASE` times before giving up. What the caller needs to know is that the write did not - land and nothing was damaged, which is the same contract as the outage above and the same - retry-later shape. ⛔ The one answer this must never be is 200: reaching this handler means the - only way to have "succeeded" was to overwrite somebody else's work. - """ - return JSONResponse( - {"error": {"code": "store_conflict", - "message": f"somebody else changed this database while your change was being " - f"saved, and it could not be merged. Nothing was saved or " - f"overwritten. Try again. {exc}"}}, - status_code=503) - - -@app.get("/api/health") -def health(): - """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not - reveal anything about the deployment beyond "the process is up". No version, no tenant list, - no config: a health endpoint is the one URL every scanner finds first. - - ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned - release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated - caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind - a session — and the authoritative copy is the `VERSION` file in the Space repo, which - `deploy_web.space_version()` reads without needing the app to be up at all.""" - return {"ok": True} - - -app.include_router(routes_auth.router) -app.include_router(routes_nav.router) -app.include_router(routes_customers.router) -app.include_router(routes_products.router) -app.include_router(routes_assets.router) -app.include_router(routes_tables.router) -app.include_router(routes_grid.router) -app.include_router(routes_records.router) -# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user -# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so -# a new page is a builder module plus one registry line, and nothing here changes. -app.include_router(routes_pages.router) -app.include_router(routes_admin.router) -app.include_router(routes_automation.router) -app.include_router(routes_keychain.router) -app.include_router(routes_statements.router) -# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. -# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path -# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a -# platform-operator route ever shadowing each other. Every path it declares is gated by -# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. -app.include_router(routes_platform_admin.router) -# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are -# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is -# something every user does with their own views/folders/databases. -app.include_router(routes_alerts.router) -app.include_router(routes_shares.router) -app.include_router(routes_uploads.router) -# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this -# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: -# a router included AFTER it answers 404 forever while importing fine, type-checking fine and -# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. -# -# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished -# features that would have shipped dead. The workers each posted a mount ask and said they would -# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The -# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the -# control is that the enumeration must NAME every router in this file.** -app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated -app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory -# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its -# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so -# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token -# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot -# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. -app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door -# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, -# because the wave-23 scar is exactly this: three finished routers shipped with no include_router -# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks -# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. -app.include_router(routes_odoo_tables.router) -# ⭐⭐ WAVE 31 · T49 / C4 — THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY. -# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in -# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by -# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts -# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name -# that lies to every network tab, log line and bug report. -# ⚠ MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar. -app.include_router(routes_connected_tables.router) -# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` -# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep -# copies), which is what lets a filtered view pick up a row created in another tab, by an automation -# or by a connector sync without re-downloading the world. -# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: -# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` -# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 -# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no -# mounted route — which is precisely the control the wave-23 scar above was written to install. -# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this -# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). -app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line -# ⛔ WAVE 31 (R10 / C5) — E's router, taken by the INTEGRATOR under W31-T08's own done-when -# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it. -# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at -# 60/61 for it. Four waves of the same defect — three routers in wave 23, four features in -# wave 29 — is why the assertion exists and why the line is not left for later. -app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's line -# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the -# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. -# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own -# gate before it front-inserted: `include_router` APPENDS, and `app.mount("/", _AppStatic(...), -# html=True)` swallows everything reached after it, so a router added below that mount answers -# **405 on POST and 404 on GET** while every one of its own tests passes. The comment at :213 states -# the rule; E's measurement is what turns it from advice into a number. -# ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in -# app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). -app.include_router(routes_query.router) # R1 / C5 — E's router, A's line -# ⭐⭐ WAVE 33 (C2) — THE SIXTH CONSECUTIVE WAVE IN WHICH THIS BLOCK IS THE THING THE PROTOCOL LOSES. -# Contract C2 is written for exactly that: a lane creating a `routes_*.py` posts an ASK, and the -# INTEGRATOR adds the line in the SAME wave. Both of these arrived that way (C's `ASK C-1`, G's -# `ASK G-1`), and `verify_api::section_w23_mounts` asserts each path in `app.openapi()["paths"]` -# with an NC that comments a mount out and goes RED. -# ⚠ Same placement rule as the line above — ABOVE the `app.mount("/", _AppStatic(...), html=True)` -# at the end of the file, never after it. -# ⭐⭐ THE PUBLISH DOOR IS MOUNTED AGAIN (2026-08-15). It was withheld for the wave-33 deploy -# because QA reproduced three HIGH defects that are armed ONLY by mounting it. All three are fixed -# in `routes_publish.py`, each at its cause rather than at its symptom: -# 1. W33-T68 — `form` LEFT `PUBLISHABLE_MODES`. A form view's rows ARE the submissions people -# sent it, so publishing one served other people's answers to anyone holding the link. A form -# still has its own public door (`#/form/`) which serves the BLANK form and never rows. -# 2. W33-T69 — `_visible_keys` returns `[]` when a view STORED a `visible` list and none of its -# keys survive, instead of falling back to the table default. The fallback still applies to a -# view that never stored one, which is what it was written for. -# 3. W33-T70 — three separate leaks closed: the rate limit no longer keys on the caller-supplied -# `x-forwarded-for` (it keys on the socket peer and counts FAILURES only, so a shared proxy -# peer cannot become one global bucket); every failing path now spends the same PBKDF2 the -# success path spends, closing the 206x timing gap; and an unknown token on the sibling GET -# answers the LOCKED shape rather than a 403, so that route stops sorting real tokens from -# fake ones for free. -# ⚠ `verify_api::section_w23_mounts` asserts this path in `app.openapi()["paths"]`, so its four -# EXPECTED reds should now go GREEN. A red here after this line means the mount broke, not the gate. -app.include_router(routes_publish.router) # R5 / C2 — C's router, A's line (the publish door) -# ⛔ NOT BEHIND `module_gate("product_data")`, WHICH IS THE WHOLE REASON IT IS A SECOND ASSET DOOR. -# `routes_assets.py` gates EVERY one of its routes on that module, so a connector logo served from -# there would 403 for any account without the product-data grant — i.e. the Connectors directory -# would lose its logos for exactly the accounts most likely to be setting a connector up (C6). -app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's line (brand marks) -# ⛔ TWO OF ITS SIX PATHS ARE UNAUTHENTICATED BY DESIGN (`/slack/events`, `/slack/interact`) and -# that is the DOOR, not an omission — Slack posts to them with no session and could not carry one. -# They are built on `routes_forms`' proven public shape: signing-secret verification, a sliding-window -# rate limit, a body cap and ONE non-oracle 403, so a bad signature cannot distinguish "no such -# workspace" from "not yours". ⚠ There is no auth middleware and no exempt-path allow-list in this -# file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why -# `verify_api` asserts the absence rather than an entry in a list that does not exist. -app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) -# ⭐⭐ WAVE 35 (R4/R5/R7, contracts C2/C3/C5) — THE STAR. Mounted in the SAME change that created -# `routes_starred.py`, which is the seventh consecutive wave in which this block is the artefact the -# protocol nearly loses — and the first in which the router's own lane also owns `main.py`, so there -# is no ask/mount pair to drop. `W35-T46` asserts every one of this wave's paths in -# `app.openapi()["paths"]` and CALLS one route from each. -# ⚠ Same placement rule as every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)`, or -# the router answers 404 on GET and 405 on POST while every one of its own tests passes. -app.include_router(routes_starred.router) # R4 / C2 — the star, counts, and record stars -# ⛔ R9 SAYS THIS ROUTE REPORTS AND NEVER ENFORCES, so mounting it cannot cut anybody off — there is -# no ceiling anywhere behind it (`usage_ledger` has no refusal in it). The one AI limit the product -# enforces is per COLUMN and is unchanged (`ai_enrich.ceiling_report`). -app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one AI meter -# ⛔ ITS TWO DOORS CARRY DIFFERENT WALLS ON PURPOSE (R8, and D-221 is the booked precedent): the -# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does -# not widen anything a tenant admin can reach — `verify_api` proves that by having one try. -app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane -# ⭐⭐ WAVE 36 (R8 / C4) — THE AGENT HARNESS FILE STORE, mounted in the SAME change that created -# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the -# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in -# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107). -# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the -# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green. -# ⛔ ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES — and that is safe -# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot -# match `/agents/x/harness`. The two routers share a prefix and no route. -app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent harness files -# ⭐⭐ WAVE 36 (R3 / R10 / C3) — THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that -# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in -# `app.openapi()["paths"]` AND calls them, and its NC comments this line out. -# ⛔ ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool. -# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header -# says why that is not a style choice. -app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views -# ⭐⭐ WAVE 37 (R14, amendment A11) — THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the -# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the -# artefact the protocol nearly loses, and the first in which the ask that carries it was itself -# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered -# it (D-366). The integrator found it reading the peer mailbox by hand. -# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the -# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is -# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line — keep it. -# ⛔ R14 put the provider behind a seam precisely so swapping Google in later is a config change: -# tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them -# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). -app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) - - -# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) -# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic -# behavior. An alias that kept the old auth would be a bypass of everything above it. -_GATE = module_gate(routes_customers.MODULE) - - -@app.get("/api/customers", deprecated=True) -def _customers_alias(session: Session = Depends(_GATE)): - return routes_customers._payload(session) - - -@app.patch("/api/customers/{pid}", deprecated=True) -def _patch_alias(pid: int, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - return routes_customers.patch_customer(pid, body, session) - - -def _assert_contract_floor(): - """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. - - This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the - canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and - the embedded host are serving two different schemas and the grid's own contract has forked — - the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup - on whatever machine is actually running. - """ - import aios_grid - base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] - if [f["key"] for f in base] != [f["key"] for f in FIELDS]: - raise RuntimeError( - "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " - "disagree on the base field set. Embed and standalone would serve different schemas. " - "Run aios-web/verify_fields_contract.py.") - - -def _startup_notes(): - """Say the two things an operator must know, once, at import — never in a response body.""" - if aios_session.EPHEMERAL_SECRET: - print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process " - "key. Sessions will not survive a restart and will not work across workers. " - "Set it in production.") - if os.environ.get("AIOS_INSECURE_SSL") == "1": - print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound " - "requests. Local development only; never in a deployed environment.") - - -_assert_contract_floor() -_startup_notes() - - -# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. -# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login -# page cannot live behind a password prompt, so the shell's own assets must be public — which they -# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. -# -# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also -# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are -# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to -# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` -# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), -# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same -# answer as any other path that is not part of the app. -# -# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); -# this guard is what makes the API safe regardless of what the bundle happens to contain. -_DEV_FIXTURES = {"sample_customers.json"} - - -class _AppStatic(StaticFiles): - async def get_response(self, path, scope): - if Path(path).name in _DEV_FIXTURES: - raise StarletteHTTPException(status_code=404, detail="Not Found") - resp = await super().get_response(path, scope) - # Vite content-hashes everything under assets/ (a change is a NEW url), so those are - # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. - # index.html must stay revalidated or a deploy would strand returning browsers on the old - # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. - # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, - # and `startswith("assets/")` silently skipped every asset (measured on the local probe). - if path.replace("\\", "/").lstrip("/").startswith("assets/"): - resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" - else: - resp.headers["Cache-Control"] = "no-cache" - return resp - - -# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets -if _WEB_DIST.is_dir(): - app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") - - -# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- -# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in -# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's -# default envelope; scoped users still pay their own scope's first build, once. -# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends -# run against fakes) can never fire a live Odoo pull. -def _seed_and_sync_store(): - """Bring the analytical store (harness.datastore) LIVE for this container — the app.py - bootstrap, mirrored (2026-07-31, owner item 1). - - `harness.datastore` powers every measure column/condition, and ONLY app.py used to call - `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store - that never existed and every measure cell served blank. Seeding alone was NOT enough either, - and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless - `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands - EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced - forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the - write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded - passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still - serve. - """ - import time as _t - t0 = _t.time() - try: - from harness import datastore as _ds - if _ds.ensure_seed(): - print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") - # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's - # search_reads must never interleave on the shared client's xmlrpc transport. - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - res = {} - for i in range(12): - res = _ds.sync_all(log=lambda *a, **k: None) - print(f"[aios-api] datastore sync pass {i + 1}: " - + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) - if all(v.get("phase") == "live" for v in res.values()): - break - # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see - # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — - # one reconcile pass at boot removes both classes of phantom row before the first - # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of - # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. - try: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception: - pass - print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " - f"(ready={_ds.ready()})") - # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. - # - # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside - # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a - # freshly booted container could spawn the four locked databases was T+30 MINUTES. The - # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s - # across a ~20-minute window over two boots returned the pre-wave schema on all 20 - # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like - # this — a thing that never ran, mistaken for a thing that ran and failed. - # - # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is - # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, - # then the delete reconcile. Calling it from the other thread would race the seed and hit - # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which - # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild - # refuses — correctly, but for a reason that reads like a data loss scare. - # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. - # ⭐⭐ W35-T45 / R11 — TENANT #0'S ODOO CREDENTIAL MOVES ONTO ITS KEYCHAIN, IN THE CONTAINER. - # ⚠ BEFORE the relational rebuild, deliberately: the rebuild resolves its connector through - # `rt.odoo_source()`, so running the migration first means the very next read already goes - # through the keychain branch and a broken migration is visible in THIS boot's log rather - # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. - _migrate_env_odoo("boot") - _pull_meta("boot") - _rebuild_odoo_relational("boot") - # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. - # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and - # this sweep is unrelated to it — putting it last means a slow Odoo sync cannot delay the - # one thing on this path that fixes a grid the owner has asked about twice. - _sweep_automation_schemas("boot") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] datastore seed/sync skipped: {e}") - - -def _migrate_env_odoo(why): - """⭐⭐ W35-T45 / R11 — move tenant #0's environment Odoo credential onto its keychain. - - ⛔⛔ IN THE CONTAINER, WHICH IS THE WHOLE REASON THIS IS A BOOT LINE AND NOT A SCRIPT. D-195, - measured three times: a developer's CLI write to the tenant store is reverted by the running - Space within a minute (download-modify-upload, last-write-wins) — and the write REPORTS SUCCESS - every time, then a fresh read confirms it, and it is gone by the next poll. A CLI migration would - be a dry run that lies, and what it would lie about here is a credential. - - ⚠ IDEMPOTENT AND SCOPED TO TENANT #0 by `routes_keychain.env_odoo_available`, so on every other - tenant and on every later boot this is one `list_entries` read and a line. - ⚠ FAIL-QUIET: this must never take a boot down. But it is never SILENT — a skip prints its reason, - because "already migrated", "no keychain key on this deployment" and "the env is incomplete" are - three different operator actions and a blank Keychain page cannot tell them apart. - """ - try: - import routes_keychain as _kc_routes - from harness import runtime as _runtime - rep = _kc_routes.migrate_env_odoo(_runtime.get_runtime("royal-imports")) - if rep.get("done"): - print(f"[aios-api] odoo credential migrated onto the keychain ({why}): " - f"entry={rep['entry']} carried_pause={rep['carried_pause']}" - + (f" PROBLEM: {rep['why']}" if rep.get("why") else "")) - else: - print(f"[aios-api] odoo keychain migration skipped ({why}): {rep.get('why')}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}") - - -def _pull_meta(why): - """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. - - ⛔ WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the - process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a - developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE - product stays empty — which is the whole difference between "the loader works" and "the - product has the data". Odoo is already arranged this way (`sync_all` runs in the container); - this is the same arrangement for the second connector. - - ⚠ FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no - error: a tenant that has not connected Meta is a normal state, and this runs on every boot. - ⚠ The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot - is not the place for a 90-day backfill — the resync pass widens it. - """ - try: - from harness import meta_store as _meta +"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. + +It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, +reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the +JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. + +WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: + + * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared + `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same + anonymous principal and no route could scope anything. Now a signed stateless cookie carries + a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the + user RECORD. `/api/health` stays unauthenticated (liveness only). + * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home + for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). + * **The SSL shim is env-gated.** It used to run at import, unconditionally. + +⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the +session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This +is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB +commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. +""" +import os +import sys +from pathlib import Path + +# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- +# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an +# unverified default context; the HF Space and the container are unaffected. This used to run +# unconditionally at import — i.e. the shipped container disabled TLS verification for every +# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a +# man-in-the-middle away from being someone else's data. It is now opt-in, must be set +# deliberately, and is never on by default. +# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. +if os.environ.get("AIOS_INSECURE_SSL") == "1": + import ssl + ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 + +# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container +# (where the layout differs from the local sibling-dir default). --- +_HERE = Path(__file__).resolve() +_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) +for _p in (str(_RI), str(_HERE.parent)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from dotenv import load_dotenv # noqa: E402 +load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed + +from fastapi import Body, Depends, FastAPI, Request # noqa: E402 +from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 +from fastapi.responses import JSONResponse # noqa: E402 +from fastapi.staticfiles import StaticFiles # noqa: E402 +from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 + +import aios_session # noqa: E402 +import routes_admin # noqa: E402 +import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) +import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) +import routes_auth # noqa: E402 +import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) +import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) +import routes_customers # noqa: E402 +import routes_grid # noqa: E402 +import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) +import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) +import routes_nav # noqa: E402 +import routes_pages # noqa: E402 +import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) +import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) +import routes_records # noqa: E402 +import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) +import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) +import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) +import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) +import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) +import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) +import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) +import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) +import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) +import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) +import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) +import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) +import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) +import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) +import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) +import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) +import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) +import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) +import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router) +from core import grid_events # noqa: E402 +# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. +# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a +# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not. +from core import data_binding # noqa: E402 +from core import store # noqa: E402 +from deps import Session, module_gate # noqa: E402 + +_WEB_DIST = _HERE.parents[1] / "web" / "dist" + +# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- +# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never +# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the +# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so +# embed == standalone by construction. +# +# TWO REASONS THIS IS HERE and not folded into the routes: +# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR +# layout is broken, and discovering that from a 500 on the first customer request instead of +# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a +# rewrite dropped it and this restores it.) +# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and +# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this +# API have not drifted. The rewrite removed them and the referee went red; retargeting the +# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do +# not delete — but only when the subject genuinely moved. Here it should not have moved). +# +# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from +# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and +# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone +# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the +# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. +import json as _json_contract # noqa: E402 +_FIELDS_PATH = _RI / "aios_grid_fields.json" +if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " + "platform root (it also carries the data layer this API imports)." + ) +_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) +FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract +# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. +# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be +# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. +_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS + if f["source"] == "odoo" and f["type"] in ("text", "status", "select", + "date")} + +app = FastAPI(title="AIOS web API") + +# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with +# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the +# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. +app.add_middleware(GZipMiddleware, minimum_size=1024) + + +@app.exception_handler(StarletteHTTPException) +def _error_shape(request: Request, exc: StarletteHTTPException): + """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. + + `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a + 422 from a malformed path param) is wrapped here so a client never has to branch on two + different error bodies. + """ + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + body = detail + else: + body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} + return JSONResponse(body, status_code=exc.status_code, + headers=getattr(exc, "headers", None)) + + +@app.exception_handler(grid_events.StoreUnavailable) +def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): + """THE SAFETY NET for "the store is down" → 503, from anywhere. + + ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a + `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own + try/except — surfaced as a 500. A store outage is a normal operational state and every route + here touches the store at least twice (the workspace read, then the write), so "remember to + wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler + means a store outage can only ever be a 503, whichever call raised it. + + The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this + adapter, which has no durable session dict to degrade into. A 200 over a write that + evaporated is the failure the whole rule exists to prevent. + """ + return JSONResponse( + {"error": {"code": "store_unavailable", + "message": "the tenant store is unavailable. No change was saved"}}, + status_code=503) + + +@app.exception_handler(data_binding.StoreWriteRefused) +def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused): + """D-315 — this deployment may not write that store. 503, with the REASON. + + ⛔ IT ECHOES THE EXCEPTION'S OWN MESSAGE, WHERE ITS NEIGHBOUR ABOVE USES A FIXED SENTENCE, and + the difference is the point. "The store is unavailable" is the whole truth about an outage; a + refusal has a cause the operator cannot otherwise discover, because per D-160 the Space + environment cannot be read from outside. A refusal that said only "unavailable" would send + somebody hunting for a network fault that does not exist ([[report-the-cause-before-you-fix-it]]). + The message names the deployment, the store and the fix, and carries no customer data. + + ⛔ AND IT IS ITS OWN HANDLER RATHER THAN A SUBCLASS OF `StoreUnavailable`. Several routes carry + `except StoreUnavailable:` blocks that DEGRADE to a session-scoped fallback workspace — correct + for an outage, catastrophic here, because the user would be told the change was saved. A + distinct type falls through every one of them to this handler. + """ + return JSONResponse( + {"error": {"code": "store_write_refused", + "message": f"this deployment may not write the tenant store. No change was " + f"saved. {exc}"}}, + status_code=503) + + +@app.exception_handler(store.StoreConflict) +def _store_conflict(request: Request, exc: store.StoreConflict): + """D-305 — the document moved under this write more times than it could be rebased. 503. + + ⚠ 503 RATHER THAN 409, DELIBERATELY. A 409 invites the client to resolve a conflict, and there + is nothing here for it to resolve: the store already re-read and re-applied this change up to + `_MAX_REBASE` times before giving up. What the caller needs to know is that the write did not + land and nothing was damaged, which is the same contract as the outage above and the same + retry-later shape. ⛔ The one answer this must never be is 200: reaching this handler means the + only way to have "succeeded" was to overwrite somebody else's work. + """ + return JSONResponse( + {"error": {"code": "store_conflict", + "message": f"somebody else changed this database while your change was being " + f"saved, and it could not be merged. Nothing was saved or " + f"overwritten. Try again. {exc}"}}, + status_code=503) + + +@app.get("/api/health") +def health(): + """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not + reveal anything about the deployment beyond "the process is up". No version, no tenant list, + no config: a health endpoint is the one URL every scanner finds first. + + ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned + release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated + caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind + a session — and the authoritative copy is the `VERSION` file in the Space repo, which + `deploy_web.space_version()` reads without needing the app to be up at all.""" + return {"ok": True} + + +app.include_router(routes_auth.router) +app.include_router(routes_nav.router) +app.include_router(routes_customers.router) +app.include_router(routes_products.router) +app.include_router(routes_assets.router) +app.include_router(routes_tables.router) +app.include_router(routes_grid.router) +app.include_router(routes_records.router) +# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user +# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so +# a new page is a builder module plus one registry line, and nothing here changes. +app.include_router(routes_pages.router) +app.include_router(routes_admin.router) +app.include_router(routes_automation.router) +app.include_router(routes_keychain.router) +app.include_router(routes_statements.router) +# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. +# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path +# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a +# platform-operator route ever shadowing each other. Every path it declares is gated by +# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. +app.include_router(routes_platform_admin.router) +# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are +# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is +# something every user does with their own views/folders/databases. +app.include_router(routes_alerts.router) +app.include_router(routes_shares.router) +app.include_router(routes_uploads.router) +# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this +# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: +# a router included AFTER it answers 404 forever while importing fine, type-checking fine and +# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. +# +# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished +# features that would have shipped dead. The workers each posted a mount ask and said they would +# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The +# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the +# control is that the enumeration must NAME every router in this file.** +app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated +app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory +# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its +# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so +# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token +# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot +# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. +app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door +# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, +# because the wave-23 scar is exactly this: three finished routers shipped with no include_router +# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks +# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. +app.include_router(routes_odoo_tables.router) +# ⭐⭐ WAVE 31 · T49 / C4 — THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY. +# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in +# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by +# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts +# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name +# that lies to every network tab, log line and bug report. +# ⚠ MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar. +app.include_router(routes_connected_tables.router) +# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` +# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep +# copies), which is what lets a filtered view pick up a row created in another tab, by an automation +# or by a connector sync without re-downloading the world. +# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: +# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` +# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 +# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no +# mounted route — which is precisely the control the wave-23 scar above was written to install. +# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this +# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). +app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line +# ⛔ WAVE 31 (R10 / C5) — E's router, taken by the INTEGRATOR under W31-T08's own done-when +# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it. +# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at +# 60/61 for it. Four waves of the same defect — three routers in wave 23, four features in +# wave 29 — is why the assertion exists and why the line is not left for later. +app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's line +# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the +# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. +# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own +# gate before it front-inserted: `include_router` APPENDS, and `app.mount("/", _AppStatic(...), +# html=True)` swallows everything reached after it, so a router added below that mount answers +# **405 on POST and 404 on GET** while every one of its own tests passes. The comment at :213 states +# the rule; E's measurement is what turns it from advice into a number. +# ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in +# app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). +app.include_router(routes_query.router) # R1 / C5 — E's router, A's line +# ⭐⭐ WAVE 33 (C2) — THE SIXTH CONSECUTIVE WAVE IN WHICH THIS BLOCK IS THE THING THE PROTOCOL LOSES. +# Contract C2 is written for exactly that: a lane creating a `routes_*.py` posts an ASK, and the +# INTEGRATOR adds the line in the SAME wave. Both of these arrived that way (C's `ASK C-1`, G's +# `ASK G-1`), and `verify_api::section_w23_mounts` asserts each path in `app.openapi()["paths"]` +# with an NC that comments a mount out and goes RED. +# ⚠ Same placement rule as the line above — ABOVE the `app.mount("/", _AppStatic(...), html=True)` +# at the end of the file, never after it. +# ⭐⭐ THE PUBLISH DOOR IS MOUNTED AGAIN (2026-08-15). It was withheld for the wave-33 deploy +# because QA reproduced three HIGH defects that are armed ONLY by mounting it. All three are fixed +# in `routes_publish.py`, each at its cause rather than at its symptom: +# 1. W33-T68 — `form` LEFT `PUBLISHABLE_MODES`. A form view's rows ARE the submissions people +# sent it, so publishing one served other people's answers to anyone holding the link. A form +# still has its own public door (`#/form/`) which serves the BLANK form and never rows. +# 2. W33-T69 — `_visible_keys` returns `[]` when a view STORED a `visible` list and none of its +# keys survive, instead of falling back to the table default. The fallback still applies to a +# view that never stored one, which is what it was written for. +# 3. W33-T70 — three separate leaks closed: the rate limit no longer keys on the caller-supplied +# `x-forwarded-for` (it keys on the socket peer and counts FAILURES only, so a shared proxy +# peer cannot become one global bucket); every failing path now spends the same PBKDF2 the +# success path spends, closing the 206x timing gap; and an unknown token on the sibling GET +# answers the LOCKED shape rather than a 403, so that route stops sorting real tokens from +# fake ones for free. +# ⚠ `verify_api::section_w23_mounts` asserts this path in `app.openapi()["paths"]`, so its four +# EXPECTED reds should now go GREEN. A red here after this line means the mount broke, not the gate. +app.include_router(routes_publish.router) # R5 / C2 — C's router, A's line (the publish door) +# ⛔ NOT BEHIND `module_gate("product_data")`, WHICH IS THE WHOLE REASON IT IS A SECOND ASSET DOOR. +# `routes_assets.py` gates EVERY one of its routes on that module, so a connector logo served from +# there would 403 for any account without the product-data grant — i.e. the Connectors directory +# would lose its logos for exactly the accounts most likely to be setting a connector up (C6). +app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's line (brand marks) +# ⛔ TWO OF ITS SIX PATHS ARE UNAUTHENTICATED BY DESIGN (`/slack/events`, `/slack/interact`) and +# that is the DOOR, not an omission — Slack posts to them with no session and could not carry one. +# They are built on `routes_forms`' proven public shape: signing-secret verification, a sliding-window +# rate limit, a body cap and ONE non-oracle 403, so a bad signature cannot distinguish "no such +# workspace" from "not yours". ⚠ There is no auth middleware and no exempt-path allow-list in this +# file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why +# `verify_api` asserts the absence rather than an entry in a list that does not exist. +app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) +# ⭐⭐ WAVE 35 (R4/R5/R7, contracts C2/C3/C5) — THE STAR. Mounted in the SAME change that created +# `routes_starred.py`, which is the seventh consecutive wave in which this block is the artefact the +# protocol nearly loses — and the first in which the router's own lane also owns `main.py`, so there +# is no ask/mount pair to drop. `W35-T46` asserts every one of this wave's paths in +# `app.openapi()["paths"]` and CALLS one route from each. +# ⚠ Same placement rule as every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)`, or +# the router answers 404 on GET and 405 on POST while every one of its own tests passes. +app.include_router(routes_starred.router) # R4 / C2 — the star, counts, and record stars +# ⛔ R9 SAYS THIS ROUTE REPORTS AND NEVER ENFORCES, so mounting it cannot cut anybody off — there is +# no ceiling anywhere behind it (`usage_ledger` has no refusal in it). The one AI limit the product +# enforces is per COLUMN and is unchanged (`ai_enrich.ceiling_report`). +app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one AI meter +# ⛔ ITS TWO DOORS CARRY DIFFERENT WALLS ON PURPOSE (R8, and D-221 is the booked precedent): the +# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does +# not widen anything a tenant admin can reach — `verify_api` proves that by having one try. +app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane +# ⭐⭐ WAVE 36 (R8 / C4) — THE AGENT HARNESS FILE STORE, mounted in the SAME change that created +# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the +# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in +# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107). +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green. +# ⛔ ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES — and that is safe +# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot +# match `/agents/x/harness`. The two routers share a prefix and no route. +app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent harness files +# ⭐⭐ WAVE 36 (R3 / R10 / C3) — THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that +# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in +# `app.openapi()["paths"]` AND calls them, and its NC comments this line out. +# ⛔ ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool. +# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header +# says why that is not a style choice. +app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views +# ⭐⭐ WAVE 37 (R14, amendment A11) — THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the +# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the +# artefact the protocol nearly loses, and the first in which the ask that carries it was itself +# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered +# it (D-366). The integrator found it reading the peer mailbox by hand. +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is +# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line — keep it. +# ⛔ R14 put the provider behind a seam precisely so swapping Google in later is a config change: +# tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them +# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). +app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) + + +# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) +# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic +# behavior. An alias that kept the old auth would be a bypass of everything above it. +_GATE = module_gate(routes_customers.MODULE) + + +@app.get("/api/customers", deprecated=True) +def _customers_alias(session: Session = Depends(_GATE)): + return routes_customers._payload(session) + + +@app.patch("/api/customers/{pid}", deprecated=True) +def _patch_alias(pid: int, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + return routes_customers.patch_customer(pid, body, session) + + +def _assert_contract_floor(): + """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. + + This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the + canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and + the embedded host are serving two different schemas and the grid's own contract has forked — + the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup + on whatever machine is actually running. + """ + import aios_grid + base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] + if [f["key"] for f in base] != [f["key"] for f in FIELDS]: + raise RuntimeError( + "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " + "disagree on the base field set. Embed and standalone would serve different schemas. " + "Run aios-web/verify_fields_contract.py.") + + +def _startup_notes(): + """Say the two things an operator must know, once, at import — never in a response body.""" + if aios_session.EPHEMERAL_SECRET: + print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process " + "key. Sessions will not survive a restart and will not work across workers. " + "Set it in production.") + if os.environ.get("AIOS_INSECURE_SSL") == "1": + print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound " + "requests. Local development only; never in a deployed environment.") + + +_assert_contract_floor() +_startup_notes() + + +# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. +# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login +# page cannot live behind a password prompt, so the shell's own assets must be public — which they +# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. +# +# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also +# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are +# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to +# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` +# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), +# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same +# answer as any other path that is not part of the app. +# +# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); +# this guard is what makes the API safe regardless of what the bundle happens to contain. +_DEV_FIXTURES = {"sample_customers.json"} + + +class _AppStatic(StaticFiles): + async def get_response(self, path, scope): + if Path(path).name in _DEV_FIXTURES: + raise StarletteHTTPException(status_code=404, detail="Not Found") + resp = await super().get_response(path, scope) + # Vite content-hashes everything under assets/ (a change is a NEW url), so those are + # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. + # index.html must stay revalidated or a deploy would strand returning browsers on the old + # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. + # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, + # and `startswith("assets/")` silently skipped every asset (measured on the local probe). + if path.replace("\\", "/").lstrip("/").startswith("assets/"): + resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" + else: + resp.headers["Cache-Control"] = "no-cache" + return resp + + +# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets +if _WEB_DIST.is_dir(): + app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") + + +# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- +# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in +# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's +# default envelope; scoped users still pay their own scope's first build, once. +# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends +# run against fakes) can never fire a live Odoo pull. +def _seed_and_sync_store(): + """Bring the analytical store (harness.datastore) LIVE for this container — the app.py + bootstrap, mirrored (2026-07-31, owner item 1). + + `harness.datastore` powers every measure column/condition, and ONLY app.py used to call + `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store + that never existed and every measure cell served blank. Seeding alone was NOT enough either, + and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless + `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands + EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced + forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the + write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded + passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still + serve. + """ + import time as _t + t0 = _t.time() + try: + from harness import datastore as _ds + if _ds.ensure_seed(): + print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") + # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's + # search_reads must never interleave on the shared client's xmlrpc transport. + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + res = {} + for i in range(12): + res = _ds.sync_all(log=lambda *a, **k: None) + print(f"[aios-api] datastore sync pass {i + 1}: " + + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) + if all(v.get("phase") == "live" for v in res.values()): + break + # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see + # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — + # one reconcile pass at boot removes both classes of phantom row before the first + # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of + # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. + try: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception: + pass + print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " + f"(ready={_ds.ready()})") + # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. + # + # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside + # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a + # freshly booted container could spawn the four locked databases was T+30 MINUTES. The + # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s + # across a ~20-minute window over two boots returned the pre-wave schema on all 20 + # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like + # this — a thing that never ran, mistaken for a thing that ran and failed. + # + # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is + # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, + # then the delete reconcile. Calling it from the other thread would race the seed and hit + # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which + # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild + # refuses — correctly, but for a reason that reads like a data loss scare. + # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. + # ⭐⭐ W35-T45 / R11 — TENANT #0'S ODOO CREDENTIAL MOVES ONTO ITS KEYCHAIN, IN THE CONTAINER. + # ⚠ BEFORE the relational rebuild, deliberately: the rebuild resolves its connector through + # `rt.odoo_source()`, so running the migration first means the very next read already goes + # through the keychain branch and a broken migration is visible in THIS boot's log rather + # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. + _migrate_env_odoo("boot") + _pull_meta("boot") + _rebuild_odoo_relational("boot") + # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. + # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and + # this sweep is unrelated to it — putting it last means a slow Odoo sync cannot delay the + # one thing on this path that fixes a grid the owner has asked about twice. + _sweep_automation_schemas("boot") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] datastore seed/sync skipped: {e}") + + +def _migrate_env_odoo(why): + """⭐⭐ W35-T45 / R11 — move tenant #0's environment Odoo credential onto its keychain. + + ⛔⛔ IN THE CONTAINER, WHICH IS THE WHOLE REASON THIS IS A BOOT LINE AND NOT A SCRIPT. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and the write REPORTS SUCCESS + every time, then a fresh read confirms it, and it is gone by the next poll. A CLI migration would + be a dry run that lies, and what it would lie about here is a credential. + + ⚠ IDEMPOTENT AND SCOPED TO TENANT #0 by `routes_keychain.env_odoo_available`, so on every other + tenant and on every later boot this is one `list_entries` read and a line. + ⚠ FAIL-QUIET: this must never take a boot down. But it is never SILENT — a skip prints its reason, + because "already migrated", "no keychain key on this deployment" and "the env is incomplete" are + three different operator actions and a blank Keychain page cannot tell them apart. + """ + try: + import routes_keychain as _kc_routes + from harness import runtime as _runtime + rep = _kc_routes.migrate_env_odoo(_runtime.get_runtime("royal-imports")) + if rep.get("done"): + print(f"[aios-api] odoo credential migrated onto the keychain ({why}): " + f"entry={rep['entry']} carried_pause={rep['carried_pause']}" + + (f" PROBLEM: {rep['why']}" if rep.get("why") else "")) + else: + print(f"[aios-api] odoo keychain migration skipped ({why}): {rep.get('why')}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}") + + +def _pull_meta(why): + """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. + + ⛔ WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the + process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a + developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE + product stays empty — which is the whole difference between "the loader works" and "the + product has the data". Odoo is already arranged this way (`sync_all` runs in the container); + this is the same arrangement for the second connector. + + ⚠ FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no + error: a tenant that has not connected Meta is a normal state, and this runs on every boot. + ⚠ The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot + is not the place for a 90-day backfill — the resync pass widens it. + """ + try: + from harness import meta_store as _meta if not _meta.token(): - print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this " - f"deployment - the connector is idle, not broken") + print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this " + f"deployment - the connector is idle, not broken") return False - # ⛔ PASSED, NOT SET IN THE ENVIRONMENT. `meta_store.INSIGHTS_DAYS` binds at - # IMPORT, so an `os.environ.setdefault` here executed after the module was - # already loaded and changed NOTHING: every boot pulled 90 days instead of 7, - # which is the slow path that trips the per-ad-account rate limit and never - # finishes. A knob read at import cannot be turned by a caller at runtime. - rep = _meta.sync("royal-imports", log=lambda *_a: None, insights_days=7) - for p in rep.get("problems") or []: - print(f"[aios-api] meta sync PROBLEM ({why}): {p}") + # ⛔ PASSED, NOT SET IN THE ENVIRONMENT. `meta_store.INSIGHTS_DAYS` binds at + # IMPORT, so an `os.environ.setdefault` here executed after the module was + # already loaded and changed NOTHING: every boot pulled 90 days instead of 7, + # which is the slow path that trips the per-ad-account rate limit and never + # finishes. A knob read at import cannot be turned by a caller at runtime. + rep = _meta.sync("royal-imports", log=lambda *_a: None, insights_days=7) + for p in rep.get("problems") or []: + print(f"[aios-api] meta sync PROBLEM ({why}): {p}") print(f"[aios-api] meta sync done ({why}): " + ", ".join(f"{k}={v['in_mirror']}" for k, v in sorted(rep["tables"].items()))) return True except Exception as e: # noqa: BLE001 print(f"[aios-api] meta sync FAILED ({why}): {type(e).__name__}: {e}") return False - - + + def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False): - """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. - - ⭐ CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than - tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just - established which tenant's file this process holds open. Spawning Meta here inherits that - binding instead of rebinding it under live readers — which is the documented way to serve one - tenant's rows to another with nothing raised. - - ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables - in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure, - and it must not print an error every 30 minutes for every tenant that does not use Meta. - ⛔ Its own try/except for the reason the two passes above have theirs: a Meta refusal must not - cancel an Odoo rebuild that already succeeded. - """ - try: - import meta_relational as _meta + """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. + + ⭐ CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than + tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just + established which tenant's file this process holds open. Spawning Meta here inherits that + binding instead of rebinding it under live readers — which is the documented way to serve one + tenant's rows to another with nothing raised. + + ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables + in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure, + and it must not print an error every 30 minutes for every tenant that does not use Meta. + ⛔ Its own try/except for the reason the two passes above have theirs: a Meta refusal must not + cancel an Odoo rebuild that already succeeded. + """ + try: + import meta_relational as _meta state = _meta.refresh_state(rt, why, previous_fingerprint=previous_fingerprint) counts = state["written"] if counts: @@ -688,178 +688,178 @@ def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations print(f"[aios-api] meta relational rebuild FAILED ({why}): {type(e).__name__}: {e}") return {"present": False, "known": False, "fingerprint": None, "changed": True, "applied": False, "written": {}} - - -def _sweep_automation_schemas(why): - """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. - - Owner items 13 and 15 — the TikTok comments lock, and the comment CONTENT column he says he has - asked for *"many times"*. **Both were already correct in the source.** `TT_COMMENT_FIELDS` - carries `field_def("text", "Comment")` and `TT_LOCKED_TABLES` already contains the comments - table, both since wave 31, with his words quoted in the comment beside them. So this ticket is - not a schema change and there is nothing to design: it is DELIVERY. - - ⛔ THE MECHANISM, WHICH IS THE WHOLE OF IT. `ut_ensure` MERGES fields into an existing table and - stamps `recordMode` — but only **when something calls it**, and the only callers are automation - runs. A tenant whose `ut_tt_comments` was spawned before the declaration changed keeps the old - shape until an automation happens to run against it. Nothing sweeps existing tenants. That is - [[a-migration-that-runs-on-the-next-write]], and it is this wave's stated thesis: a declaration - that never reaches a tenant is indistinguishable from one that was never written. - - ⛔⛔ AND IT MUST RUN **IN THE CONTAINER**, WHICH IS WHY THIS IS IN `main.py` AND NOT A SCRIPT. - D-195, measured three times: a developer's CLI write to the tenant store is reverted by the - running Space within a minute (download-modify-upload, last-write-wins) — and **the write - reports success every time**, then a FRESH read confirms it, and it is gone by the next poll. - A connector's tables must be spawned BY THE CONTAINER; a CLI spawn is a dry run that lies. - - ⚠ EVERY TENANT, unlike `_rebuild_odoo_relational` below — and the asymmetry is deliberate - rather than an oversight. That function is scoped to royal because it derives from the DuckDB - mirror, and `harness/datastore` is a ONE-FILE-AT-A-TIME process global (D-29): rebinding it per - tenant in a daemon thread can serve one tenant's rows to another with nothing raised. This - sweep touches only `user_tables` through each tenant's own `rt`, which has no such global — so - the hazard that scopes that one does not exist here, and TikTok automations run in tenants - other than #0. - - ⚠ CHEAP ON A CORRECT TENANT: `ut_ensure` short-circuits when nothing changed, so this is a read - per child table on a tenant that is already right, and the whole delivery on one that is not. - ⚠ ONE TENANT'S FAILURE MUST NOT STOP THE NEXT. Each is wrapped: a tenant whose store is - unreachable at boot is reported and skipped, never allowed to abort the sweep for everyone. - """ - try: - import automation_engine as _eng - from harness import runtime as _runtime - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED. Import failed: {e}") - return - try: - schemas = _eng.platform_schemas() - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED. No declarations: {e}") - return - tenants = [] - try: - tenants = _runtime.known_tenants() - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED. Tenant list unreadable: {e}") - return - for slug in tenants: - try: - rt = _runtime.get_runtime(slug) - # ⛔⛔ FIXED 2026-08-13 — THIS SWEEP WAS SPAWNING EIGHT DATABASES IN EVERY TENANT. - # Owner: *"Database for Royal Imports, why we have fucking IG and TIktok databases."* - # `ut_ensure`'s first line is *"Create the table if it is missing"*, and this loop fed - # it every child of every platform schema for every tenant — so tenant #0, a floral and - # giftware importer with ZERO automations, woke up on 2026-08-13 at 12:05 UTC owning - # `ut_ig_posts`, `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots` and the - # four TikTok twins, all empty, all in his nav flyout. **The bug is one word wide:** - # this function's own title says *"MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE - # THE TABLES"* and its body called a CREATE-OR-MERGE function to do a MERGE-ONLY job. - # [[reuse-and-delete-are-hypotheses]] — `ut_ensure` was the right function for the - # merge and brought a second behaviour nobody wanted with it. - # - # ⭐ THE PREDICATE IS "DOES THIS TENANT ALREADY HAVE THE TABLE", read ONCE per tenant - # rather than per child — `rt.get` deep-copies the whole tenant document (28.6 MB on - # tenant #0), so asking eight times is eight copies to answer one question. - # ⚠ AND IT MUST NOT WEAKEN THE DELIVERY: a tenant that HAS `ut_tt_comments` still gets - # the `text` column and the `recordMode` stamp, which is the entire point of T07. The - # sweep now delivers to tables that exist and mints none, which is what it always - # claimed to do. - have = set(rt.get("user_tables") or {}) - ensured, skipped = [], [] - for s in schemas: - for key, child in (s.get("children") or {}).items(): - if key not in have: - skipped.append(key) - continue - got = _eng.ut_ensure(rt, child["label"], child["fields"], "automation", - key=key, lock_fields=True, - record_mode=child["record_mode"]) - if got: - ensured.append(got) - if skipped: - # ⭐ SAID OUT LOUD, never silently skipped — this repo's "no silent caps" rule. A - # sweep that quietly does nothing looks identical to a sweep that is not running, - # which is how the previous behaviour survived review in the first place. - print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " - f"present in this tenant, so nothing was created for them " - f"({', '.join(sorted(skipped))}). They are spawned by an automation that " - f"needs them, never by this sweep") - # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. - # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and - # T07's `done-when` asserts both ("no `ut_tt_*` grid carries an Instagram column"). On a - # tenant that ran TikTok before W30-T08 the 26 machine-authored IG columns are still - # there — the detector was fixed, the damage never was. - # ⚠ `kept` IS THE HONEST HALF: a foreign column that HOLDS DATA is REPORTED, never - # deleted. If it is non-empty, the screenshot shows a column and the report is the - # answer (W30/R6's second sentence). - st = {} - try: - st = _eng.retract_foreign_presets(rt, log=lambda *a, **k: None) or {} - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) {slug}: retraction FAILED: {e}") - print(f"[aios-api] schema sweep ({why}) {slug}: ensured={len(ensured)} " - f"tables={st.get('tables', 0)} columns={st.get('columns', 0)} " - f"cells={st.get('cells', 0)} flags={st.get('flags', 0)} " - f"kept={st.get('kept') or []}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) {slug}: FAILED: {e}") - # ⭐ A SUCCESS MARKER, for `_rebuild_odoo_relational`'s stated reason: D-107 was chased for a - # day on ABSENT log markers, which cannot tell "it ran and was fine" from "it was never - # reached". Three failure markers and no success marker makes silence ambiguous. - print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)") - - + + +def _sweep_automation_schemas(why): + """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. + + Owner items 13 and 15 — the TikTok comments lock, and the comment CONTENT column he says he has + asked for *"many times"*. **Both were already correct in the source.** `TT_COMMENT_FIELDS` + carries `field_def("text", "Comment")` and `TT_LOCKED_TABLES` already contains the comments + table, both since wave 31, with his words quoted in the comment beside them. So this ticket is + not a schema change and there is nothing to design: it is DELIVERY. + + ⛔ THE MECHANISM, WHICH IS THE WHOLE OF IT. `ut_ensure` MERGES fields into an existing table and + stamps `recordMode` — but only **when something calls it**, and the only callers are automation + runs. A tenant whose `ut_tt_comments` was spawned before the declaration changed keeps the old + shape until an automation happens to run against it. Nothing sweeps existing tenants. That is + [[a-migration-that-runs-on-the-next-write]], and it is this wave's stated thesis: a declaration + that never reaches a tenant is indistinguishable from one that was never written. + + ⛔⛔ AND IT MUST RUN **IN THE CONTAINER**, WHICH IS WHY THIS IS IN `main.py` AND NOT A SCRIPT. + D-195, measured three times: a developer's CLI write to the tenant store is reverted by the + running Space within a minute (download-modify-upload, last-write-wins) — and **the write + reports success every time**, then a FRESH read confirms it, and it is gone by the next poll. + A connector's tables must be spawned BY THE CONTAINER; a CLI spawn is a dry run that lies. + + ⚠ EVERY TENANT, unlike `_rebuild_odoo_relational` below — and the asymmetry is deliberate + rather than an oversight. That function is scoped to royal because it derives from the DuckDB + mirror, and `harness/datastore` is a ONE-FILE-AT-A-TIME process global (D-29): rebinding it per + tenant in a daemon thread can serve one tenant's rows to another with nothing raised. This + sweep touches only `user_tables` through each tenant's own `rt`, which has no such global — so + the hazard that scopes that one does not exist here, and TikTok automations run in tenants + other than #0. + + ⚠ CHEAP ON A CORRECT TENANT: `ut_ensure` short-circuits when nothing changed, so this is a read + per child table on a tenant that is already right, and the whole delivery on one that is not. + ⚠ ONE TENANT'S FAILURE MUST NOT STOP THE NEXT. Each is wrapped: a tenant whose store is + unreachable at boot is reported and skipped, never allowed to abort the sweep for everyone. + """ + try: + import automation_engine as _eng + from harness import runtime as _runtime + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED. Import failed: {e}") + return + try: + schemas = _eng.platform_schemas() + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED. No declarations: {e}") + return + tenants = [] + try: + tenants = _runtime.known_tenants() + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED. Tenant list unreadable: {e}") + return + for slug in tenants: + try: + rt = _runtime.get_runtime(slug) + # ⛔⛔ FIXED 2026-08-13 — THIS SWEEP WAS SPAWNING EIGHT DATABASES IN EVERY TENANT. + # Owner: *"Database for Royal Imports, why we have fucking IG and TIktok databases."* + # `ut_ensure`'s first line is *"Create the table if it is missing"*, and this loop fed + # it every child of every platform schema for every tenant — so tenant #0, a floral and + # giftware importer with ZERO automations, woke up on 2026-08-13 at 12:05 UTC owning + # `ut_ig_posts`, `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots` and the + # four TikTok twins, all empty, all in his nav flyout. **The bug is one word wide:** + # this function's own title says *"MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE + # THE TABLES"* and its body called a CREATE-OR-MERGE function to do a MERGE-ONLY job. + # [[reuse-and-delete-are-hypotheses]] — `ut_ensure` was the right function for the + # merge and brought a second behaviour nobody wanted with it. + # + # ⭐ THE PREDICATE IS "DOES THIS TENANT ALREADY HAVE THE TABLE", read ONCE per tenant + # rather than per child — `rt.get` deep-copies the whole tenant document (28.6 MB on + # tenant #0), so asking eight times is eight copies to answer one question. + # ⚠ AND IT MUST NOT WEAKEN THE DELIVERY: a tenant that HAS `ut_tt_comments` still gets + # the `text` column and the `recordMode` stamp, which is the entire point of T07. The + # sweep now delivers to tables that exist and mints none, which is what it always + # claimed to do. + have = set(rt.get("user_tables") or {}) + ensured, skipped = [], [] + for s in schemas: + for key, child in (s.get("children") or {}).items(): + if key not in have: + skipped.append(key) + continue + got = _eng.ut_ensure(rt, child["label"], child["fields"], "automation", + key=key, lock_fields=True, + record_mode=child["record_mode"]) + if got: + ensured.append(got) + if skipped: + # ⭐ SAID OUT LOUD, never silently skipped — this repo's "no silent caps" rule. A + # sweep that quietly does nothing looks identical to a sweep that is not running, + # which is how the previous behaviour survived review in the first place. + print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " + f"present in this tenant, so nothing was created for them " + f"({', '.join(sorted(skipped))}). They are spawned by an automation that " + f"needs them, never by this sweep") + # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. + # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and + # T07's `done-when` asserts both ("no `ut_tt_*` grid carries an Instagram column"). On a + # tenant that ran TikTok before W30-T08 the 26 machine-authored IG columns are still + # there — the detector was fixed, the damage never was. + # ⚠ `kept` IS THE HONEST HALF: a foreign column that HOLDS DATA is REPORTED, never + # deleted. If it is non-empty, the screenshot shows a column and the report is the + # answer (W30/R6's second sentence). + st = {} + try: + st = _eng.retract_foreign_presets(rt, log=lambda *a, **k: None) or {} + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) {slug}: retraction FAILED: {e}") + print(f"[aios-api] schema sweep ({why}) {slug}: ensured={len(ensured)} " + f"tables={st.get('tables', 0)} columns={st.get('columns', 0)} " + f"cells={st.get('cells', 0)} flags={st.get('flags', 0)} " + f"kept={st.get('kept') or []}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) {slug}: FAILED: {e}") + # ⭐ A SUCCESS MARKER, for `_rebuild_odoo_relational`'s stated reason: D-107 was chased for a + # day on ABSENT log markers, which cannot tell "it ran and was fine" from "it was never + # reached". Three failure markers and no success marker makes silence ambiguous. + print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)") + + def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None): - """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or - 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone - could have asked" are different diagnoses. - - ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased - for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran - and was fine" from "the rebuild was never reached". Three failure markers and no success - marker means silence is ambiguous; now it is not. - - ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The - caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and - royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks - straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME - process-global, and rebinding it in a daemon thread under live readers can serve one tenant's - rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. - """ - try: - import odoo_relational as _rel - from harness import runtime as _runtime + """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or + 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone + could have asked" are different diagnoses. + + ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased + for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran + and was fine" from "the rebuild was never reached". Three failure markers and no success + marker means silence is ambiguous; now it is not. + + ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The + caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and + royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks + straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME + process-global, and rebinding it in a daemon thread under live readers can serve one tenant's + rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. + """ + try: + import odoo_relational as _rel + from harness import runtime as _runtime if not _rel.is_royal("royal-imports"): return False - _rt = _runtime.get_runtime("royal-imports") - counts = _rel.refresh(_rt, "royal-imports") - # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. - # - # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field - # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to - # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a - # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every - # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the - # tables simply look finished and answer nothing. - # - # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count - # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. - # Calling the inner one here would return a plausible number and change no cell. - # - # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source - # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest - # refusal silently cancel a relational pass that had already succeeded. - try: - import automation_engine as _engine - _engine.refresh_relations(_rt, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") - try: - import rollup_sql as _rollup - for _b, _key, _l, _f in _rel.TABLES: - _rollup.compute(_rt, _key) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") + _rt = _runtime.get_runtime("royal-imports") + counts = _rel.refresh(_rt, "royal-imports") + # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. + # + # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field + # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to + # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a + # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every + # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the + # tables simply look finished and answer nothing. + # + # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count + # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. + # Calling the inner one here would return a plausible number and change no cell. + # + # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source + # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest + # refusal silently cancel a relational pass that had already succeeded. + try: + import automation_engine as _engine + _engine.refresh_relations(_rt, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") + try: + import rollup_sql as _rollup + for _b, _key, _l, _f in _rel.TABLES: + _rollup.compute(_rt, _key) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") print(f"[aios-api] odoo relational rebuild done ({why}): " + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) meta_state = _rebuild_meta_relational( @@ -870,11 +870,11 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out # continue. This return value answers the narrower resync-policy question: did the Odoo # table refresh itself complete, so a confirmed zero-change pass may be skipped? return True - except Exception as e: # noqa: BLE001 - # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only - # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a - # `BinderException` about a missing column is a different action from a timeout or an auth - # failure, and the text alone often does not say which it was. + except Exception as e: # noqa: BLE001 + # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only + # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a + # `BinderException` about a missing column is a different action from a timeout or an auth + # failure, and the text alone often does not say which it was. print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") return False @@ -934,84 +934,84 @@ def _meta_rebuild_force_reason(pull_confirmed, last_fingerprint, last_applied_at def _store_resync_loop(): - """Keep the analytical mirror current — the API-process stand-in for the re-sync the - Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a - refreshed pool re-reads the freshly synced store.""" + """Keep the analytical mirror current — the API-process stand-in for the re-sync the + Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a + refreshed pool re-reads the freshly synced store.""" import time as _t passes = 0 last_relational_success_at = None last_meta_fingerprint = None last_meta_applied_at = None while True: - # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. - # - # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 — - # the 30m/1h/4h/daily/manual presets, the server-side clamp, the config door and - # `odoo_relational.sync_seconds()` which turns the stored preset into seconds — and its own - # ticket said the one-line change belonged to A because `main.py` is A's file. That ASK was - # never sent, so the value was stored, displayed, clamped and IGNORED: a person could pick - # "every 4 hours" and the loop would keep resyncing every 30 minutes with nothing anywhere - # reporting the disagreement. Found by `verify_reachability` naming `sync_seconds` as a - # function whose ONLY caller was its own gate [[artifact-with-no-importer]]. - # - # ⚠ `None` MEANS MANUAL AND MUST NOT MEAN ZERO. R11's `manual` preset returns None from - # `sync_seconds`; treating that as a falsy interval would spin this loop with no sleep at - # all. It parks at the default cadence instead and simply does no work — the tenant asked - # not to be synced automatically, not for the server to stop breathing. - # ⚠ RE-READ EVERY PASS, deliberately: a cadence changed in Settings takes effect on the - # next cycle rather than at the next container restart, which is what makes it a setting. - # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is - # enforced in `sync_seconds` as well as at the write door, for the same reason. - # ⛔⛔ WAVE 34 · W34-T46 / D-246 — THE PARAGRAPH ABOVE DESCRIBED A BRANCH THAT DID NOT - # EXIST, AND THAT IS WHY NOBODY LOOKED. It said `manual` "parks at the default cadence - # instead and simply DOES NO WORK". The first half was true; the second half was not - # implemented anywhere: `_secs is None` set the sleep to 1800 and then fell into an - # UNCONDITIONAL `sync_all()`. So a tenant who chose "manual" was synced every thirty - # minutes exactly like everyone else, with the setting stored, displayed, clamped and - # obeyed by nothing — the identical failure this block's own W32/R11 note is about, one - # layer further in. A comment asserting a fix is why the second reader stops reading. - # ⚠ FIXED, AND THE FIX IS NARROW ON PURPOSE. `manual` is an ODOO connector setting, so it - # skips the Odoo mirror sync and its reconcile and NOTHING ELSE: the Meta pull and the - # relational rebuild below still run, because a tenant asking not to have Odoo polled has - # not asked for the other connector to stop. - # ⚠ AND IT STILL SLEEPS AND STILL LOOPS. The config is re-read every pass, so switching - # back off `manual` takes effect on the next cycle rather than at the next restart. - _every = 1800 - _manual = False - try: - import odoo_relational as _rel_cad - from harness import runtime as _rt_cad - # ⚠ D-245, KNOWN AND NOT FIXED HERE: the slug is hardcoded, so this reads TENANT #0's - # cadence and applies it to a loop that syncs the shared mirror. Correct while one - # tenant has Odoo; wrong the moment a second one does. Out of this ticket's scope and - # named rather than silently inherited. - _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) - _manual = _secs is None - _every = 1800 if _secs is None else max(int(_secs), 60) - except Exception: # noqa: BLE001 - pass - _t.sleep(_every) - passes += 1 + # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. + # + # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 — + # the 30m/1h/4h/daily/manual presets, the server-side clamp, the config door and + # `odoo_relational.sync_seconds()` which turns the stored preset into seconds — and its own + # ticket said the one-line change belonged to A because `main.py` is A's file. That ASK was + # never sent, so the value was stored, displayed, clamped and IGNORED: a person could pick + # "every 4 hours" and the loop would keep resyncing every 30 minutes with nothing anywhere + # reporting the disagreement. Found by `verify_reachability` naming `sync_seconds` as a + # function whose ONLY caller was its own gate [[artifact-with-no-importer]]. + # + # ⚠ `None` MEANS MANUAL AND MUST NOT MEAN ZERO. R11's `manual` preset returns None from + # `sync_seconds`; treating that as a falsy interval would spin this loop with no sleep at + # all. It parks at the default cadence instead and simply does no work — the tenant asked + # not to be synced automatically, not for the server to stop breathing. + # ⚠ RE-READ EVERY PASS, deliberately: a cadence changed in Settings takes effect on the + # next cycle rather than at the next container restart, which is what makes it a setting. + # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is + # enforced in `sync_seconds` as well as at the write door, for the same reason. + # ⛔⛔ WAVE 34 · W34-T46 / D-246 — THE PARAGRAPH ABOVE DESCRIBED A BRANCH THAT DID NOT + # EXIST, AND THAT IS WHY NOBODY LOOKED. It said `manual` "parks at the default cadence + # instead and simply DOES NO WORK". The first half was true; the second half was not + # implemented anywhere: `_secs is None` set the sleep to 1800 and then fell into an + # UNCONDITIONAL `sync_all()`. So a tenant who chose "manual" was synced every thirty + # minutes exactly like everyone else, with the setting stored, displayed, clamped and + # obeyed by nothing — the identical failure this block's own W32/R11 note is about, one + # layer further in. A comment asserting a fix is why the second reader stops reading. + # ⚠ FIXED, AND THE FIX IS NARROW ON PURPOSE. `manual` is an ODOO connector setting, so it + # skips the Odoo mirror sync and its reconcile and NOTHING ELSE: the Meta pull and the + # relational rebuild below still run, because a tenant asking not to have Odoo polled has + # not asked for the other connector to stop. + # ⚠ AND IT STILL SLEEPS AND STILL LOOPS. The config is re-read every pass, so switching + # back off `manual` takes effect on the next cycle rather than at the next restart. + _every = 1800 + _manual = False + try: + import odoo_relational as _rel_cad + from harness import runtime as _rt_cad + # ⚠ D-245, KNOWN AND NOT FIXED HERE: the slug is hardcoded, so this reads TENANT #0's + # cadence and applies it to a loop that syncs the shared mirror. Correct while one + # tenant has Odoo; wrong the moment a second one does. Out of this ticket's scope and + # named rather than silently inherited. + _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) + _manual = _secs is None + _every = 1800 if _secs is None else max(int(_secs), 60) + except Exception: # noqa: BLE001 + pass + _t.sleep(_every) + passes += 1 sync_report = None delete_report = None try: from harness import datastore as _ds - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - # ⚠ A PLAIN BRANCH, NOT AN EARLY EXIT AND NOT AN EXCEPTION. A `raise` here would be - # caught by this block's own `except` and printed as "store resync failed", turning a - # setting a person chose into a recurring error in the log; a `continue` would skip - # the Meta pull and the relational rebuild below, which `manual` says nothing about. + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + # ⚠ A PLAIN BRANCH, NOT AN EARLY EXIT AND NOT AN EXCEPTION. A `raise` here would be + # caught by this block's own `except` and printed as "store resync failed", turning a + # setting a person chose into a recurring error in the log; a `continue` would skip + # the Meta pull and the relational rebuild below, which `manual` says nothing about. if not _manual: sync_report = _ds.sync_all(log=lambda *a, **k: None) - # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot - # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; - # without it deleted Odoo lines inflate every sum on the mirror FOREVER. - # ⚠ INSIDE the branch: a reconcile is a pass over the mirror this loop was just - # told not to advance. + # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot + # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; + # without it deleted Odoo lines inflate every sum on the mirror FOREVER. + # ⚠ INSIDE the branch: a reconcile is a pass over the mirror this loop was just + # told not to advance. if passes % 4 == 0: delete_report = _ds.reconcile_deletes(log=lambda *a, **k: None) except Exception as e: # noqa: BLE001 @@ -1022,32 +1022,32 @@ def _store_resync_loop(): # refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report # is the W39 exception; deriving before a changed mirror advances would publish a one-cycle # stale worklist every single time. - # + # # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must # not swallow the sync's error message, and — worse the other way — a sync failure becomes # unconfirmed evidence and forces a recovery rebuild. Two independent failures, two - # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) - # - # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` - # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so - # "the wiring exists" and "the data is current" are different claims and only the stamp - # can tell a user which one they are looking at. - # - # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which - # is what made the boot path silent for 30 minutes after every restart; it is now the - # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here - # would be a second thing to keep in step, and the two would answer differently on the - # next ruling — the exact shape the Views top-up was just fixed for on the other side of - # this wave. - # ⛔ THE META PULL RIDES THE RESYNC TOO, AND LEAVING IT OUT WAS A REAL GAP — caught by - # reading the deploy's own boot log rather than by any gate. `_pull_meta` was wired into - # `_seed_and_sync_store` ALONE, i.e. it ran exactly once per container, at boot, behind a - # full Odoo sync. So a boot where the Graph call was rate-limited, slow or simply after the - # thread died left the mirror empty with NOTHING to retry it: the relational rebuild below - # would then find no `meta_*` tables every 30 minutes forever and skip, silently and - # correctly. A connector that can only ever be established at boot is one bad boot away - # from being permanently absent. - # ⚠ Cheap when there is nothing to do: no token => one line and return. + # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) + # + # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` + # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so + # "the wiring exists" and "the data is current" are different claims and only the stamp + # can tell a user which one they are looking at. + # + # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which + # is what made the boot path silent for 30 minutes after every restart; it is now the + # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here + # would be a second thing to keep in step, and the two would answer differently on the + # next ruling — the exact shape the Views top-up was just fixed for on the other side of + # this wave. + # ⛔ THE META PULL RIDES THE RESYNC TOO, AND LEAVING IT OUT WAS A REAL GAP — caught by + # reading the deploy's own boot log rather than by any gate. `_pull_meta` was wired into + # `_seed_and_sync_store` ALONE, i.e. it ran exactly once per container, at boot, behind a + # full Odoo sync. So a boot where the Graph call was rate-limited, slow or simply after the + # thread died left the mirror empty with NOTHING to retry it: the relational rebuild below + # would then find no `meta_*` tables every 30 minutes forever and skip, silently and + # correctly. A connector that can only ever be established at boot is one bad boot away + # from being permanently absent. + # ⚠ Cheap when there is nothing to do: no token => one line and return. _meta_pull_confirmed = _pull_meta("resync") _meta_now = _t.time() _meta_reason = _meta_rebuild_force_reason( @@ -1078,96 +1078,96 @@ def _store_resync_loop(): # Unknown is never carried forward as equality evidence. last_meta_fingerprint = None last_meta_applied_at = None - - -def _prewarm(): - import time as _t - t0 = _t.time() + + +def _prewarm(): + import time as _t + t0 = _t.time() # The Odoo seed/sync can run for minutes and is separate from the finite cache warm below. # Keep it opt-in: Live Pg serves its seeded analytical mirror at boot and lets explicit # workers perform real sync work, so a routine restart can still scale Neon back to zero. if os.environ.get("AIOS_BOOT_SYNC") == "1": import threading as _th _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() - try: - from harness import runtime as _runtime - rt = _runtime.get_runtime("royal-imports") - routes_customers.warm_default(rt) - import pages as _pages - _pages.warm_default(rt) - # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent - # from this list, so its memo was always filled by a visitor rather than by boot: call 1 of - # `GET /automations` after every deploy downloaded the whole `user_tables` document (35.8 MB - # ceiling, under the store lock) onto whoever clicked first. Memoising the WARM path — two - # waves of it — could not touch that, because the cold call is the one that fills the memo. - # ⚠ It elects a STRING and keeps no document; see `warm_default`'s own note on why caching - # the bucket would trade a latency for memory this tier does not have. - import routes_automation as _rauto - _rauto.warm_default(rt) - print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") - except Exception as e: # noqa: BLE001 — boot must not die on a warm-up - print(f"[aios-api] prewarm skipped: {e}") - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W31-T46 / D-160 — THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was -# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of -# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`: -# no `royal.duckdb` ⇒ `datastore.ready()` False forever ⇒ `ro_con()` refuses ⇒ every measure -# column blank AND — since wave 30 put the Odoo grids on the mirror — **two user-facing grids -# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the -# symptom was read as a pinned tag and a store problem for days. -# -# ⛔ AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1` -# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) — but that -# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard -# written for the dangerous case that the ordinary case walks straight past. -# -# ⭐ SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its -# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()` -# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file -# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half — -# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop — stays exactly where it was, -# because the env gate's stated reason is still true: importing `api.main` in a gate must never -# fire a live Odoo pull. -# -# ⚠ THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only -# on a genuinely fresh disk — so on every developer box and in every gate run the thread is never -# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids. -#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a -#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause. -MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""} - - + try: + from harness import runtime as _runtime + rt = _runtime.get_runtime("royal-imports") + routes_customers.warm_default(rt) + import pages as _pages + _pages.warm_default(rt) + # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent + # from this list, so its memo was always filled by a visitor rather than by boot: call 1 of + # `GET /automations` after every deploy downloaded the whole `user_tables` document (35.8 MB + # ceiling, under the store lock) onto whoever clicked first. Memoising the WARM path — two + # waves of it — could not touch that, because the cold call is the one that fills the memo. + # ⚠ It elects a STRING and keeps no document; see `warm_default`'s own note on why caching + # the bucket would trade a latency for memory this tier does not have. + import routes_automation as _rauto + _rauto.warm_default(rt) + print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") + except Exception as e: # noqa: BLE001 — boot must not die on a warm-up + print(f"[aios-api] prewarm skipped: {e}") + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W31-T46 / D-160 — THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was +# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of +# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`: +# no `royal.duckdb` ⇒ `datastore.ready()` False forever ⇒ `ro_con()` refuses ⇒ every measure +# column blank AND — since wave 30 put the Odoo grids on the mirror — **two user-facing grids +# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the +# symptom was read as a pinned tag and a store problem for days. +# +# ⛔ AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1` +# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) — but that +# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard +# written for the dangerous case that the ordinary case walks straight past. +# +# ⭐ SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its +# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()` +# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file +# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half — +# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop — stays exactly where it was, +# because the env gate's stated reason is still true: importing `api.main` in a gate must never +# fire a live Odoo pull. +# +# ⚠ THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only +# on a genuinely fresh disk — so on every developer box and in every gate run the thread is never +# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids. +#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a +#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause. +MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""} + + def _seed_mirror_if_absent(): - """Hydrate the analytical mirror when this container has none — INDEPENDENT of `AIOS_PREWARM`. - - Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no - HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator - actions, and a blank grid cannot tell them apart. - """ - from harness import datastore as _ds - MIRROR_SEED["attempted"] = True - try: - if _ds.ensure_seed(): - MIRROR_SEED["seeded"] = True - print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)") - return True - if not _ds.DB_PATH.exists(): - MIRROR_SEED["cause"] = ( - "this container has no analytical mirror and the seed snapshot could not be " - "fetched (no HF_TOKEN, or the dataset was unreachable)") - MIRROR_SEED["recommendation"] = ( - "set HF_TOKEN on the deployment; until then every connected grid and every " - "measure column served from the mirror is empty") - print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}") - except Exception as e: # noqa: BLE001 — boot must not die - MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}" - MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability" - print(f"[aios-api] mirror seed skipped: {e}") + """Hydrate the analytical mirror when this container has none — INDEPENDENT of `AIOS_PREWARM`. + + Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no + HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator + actions, and a blank grid cannot tell them apart. + """ + from harness import datastore as _ds + MIRROR_SEED["attempted"] = True + try: + if _ds.ensure_seed(): + MIRROR_SEED["seeded"] = True + print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)") + return True + if not _ds.DB_PATH.exists(): + MIRROR_SEED["cause"] = ( + "this container has no analytical mirror and the seed snapshot could not be " + "fetched (no HF_TOKEN, or the dataset was unreachable)") + MIRROR_SEED["recommendation"] = ( + "set HF_TOKEN on the deployment; until then every connected grid and every " + "measure column served from the mirror is empty") + print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}") + except Exception as e: # noqa: BLE001 — boot must not die + MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}" + MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability" + print(f"[aios-api] mirror seed skipped: {e}") return False @@ -1197,14 +1197,14 @@ def _provision_qa_sandbox(pg=None): except Exception as e: # noqa: BLE001 — boot must not die print(f"[aios-api] qa-b sandbox schema not provisioned: {type(e).__name__}: {e}") return False - - -try: - from harness import datastore as _ds_boot - if not _ds_boot.DB_PATH.exists(): - import threading as _threading_seed - _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True, - name="mirror-seed").start() + + +try: + from harness import datastore as _ds_boot + if not _ds_boot.DB_PATH.exists(): + import threading as _threading_seed + _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True, + name="mirror-seed").start() except Exception as e: # noqa: BLE001 print(f"[aios-api] mirror seed not scheduled: {e}") diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 94b0837a424d76222f93e85423413a886fad3f82..0bfd7f34e7a6fd89f55df2d5ee94463b7a45710e 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -1,402 +1,402 @@ -"""odoo_relational.py — Odoo entities as LOCKED relational databases. - -Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset -Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually -replace. - -⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the -owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports -tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the -partners who owed money — so most Odoo ids were simply absent, and the missing rows were the -reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: - - ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a - posted customer document - ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code - ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund - ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order - -⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's -own `plan()` refused above it — but the cap was never a property of the store (see the measured -banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). -The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until -wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory -budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). -The four tables together are -17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split -is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); -they are answered by the read-through rollup, which never copies a row. - -⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts -GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables -dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and -carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner -holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. -Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you -filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps -excluding, because `modules/ar` is its oracle and an oracle answers ONE question. - -⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over -ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate -**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside -`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the -`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the -wider link would count 31,418 documents and call them open invoices, so those two rollups carry -the oracle's predicate as an explicit `payment_state` condition pair. -""" -import datetime as _dt - -#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: -#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. -RI_SLUGS = ("", "royal-imports") - -INVOICES_KEY = "ut_odoo_invoices" -CUSTOMERS_KEY = "ut_odoo_customers" -ORDERS_KEY = "ut_odoo_orders" -PRODUCTS_KEY = "ut_odoo_products" -#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, -#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to -#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). -AGENTS_KEY = "ut_odoo_agents" -ACCOUNTS_KEY = "ut_odoo_accounts" -BILLS_KEY = "ut_odoo_bills" -VENDORS_KEY = "ut_odoo_vendors" - -#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. -#: -#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it -#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced -#: rather than left standing beside its own contradiction. The obstacle was never the number of -#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED -#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) -#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 -#: settles what that means: *"there is no cap in how many data from the API source … can be pulled -#: into the app"*, so the answer is a different residency, never a bigger ceiling. -#: -#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the -#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this -#: database stores no rows HERE", which is a different statement from `None` ("connected and -#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator -#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work -#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with -#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working -#: grid; that is the whole shape of the conversion. -ORDER_LINES_KEY = "ut_odoo_order_lines" -GL_LINES_KEY = "ut_odoo_gl_lines" - -#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). -JOIN_KEY = "partner_id" -#: The product-grain equivalent. -PRODUCT_JOIN_KEY = "product_id" -#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but -#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the -#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. -AGENT_JOIN_KEY = "agent_id" -#: A vendor is also a `res.partner`; same reasoning, its own column. -VENDOR_JOIN_KEY = "vendor_id" -ACCOUNT_JOIN_KEY = "account_code" - -#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two -#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. -_AR_OPEN = "payment_state IN ('not_paid','partial')" -_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" -_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" -_CONFIRMED = "state IN ('sale','done')" - -#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. -#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that -#: left the population, which is right — a reversed invoice must not keep inflating a total. But -#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded -#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible -#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. -MAX_SHRINK = 0.5 - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _registry(): - """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the - route layer before `platform/` is necessarily on the path.""" - import core.registry as registry - return registry - - -def _iso_today(): - return _dt.date.today().strftime("%Y-%m-%d") - - -def _preset(field, flow="odoo_relational"): - """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the - grid renders it grey and the preset walls refuse a rename or a delete.""" - field = dict(field) - field["automation"] = {"flowId": flow, "preset": True} - return field - - -#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. -#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing -#: one here would be a second condition vocabulary beside `_clean_rollup`'s. -_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, - {"field": "payment_state", "op": "eq", "value": "partial"}], - "conditionConj": "or"} - - -# --------------------------------------------------------------------------------------------- -# FIELD CONTRACTS -# --------------------------------------------------------------------------------------------- -# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None -# for an unknown one — which DELETES the column silently on the next read rather than erroring. -def _scope_field(): - return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", - "source": "overlay", "default": False, - "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " - "metric in this product excludes. The row is kept so no Odoo id is " - "missing; filter on this column to reconcile against the AR page."} - - -def _refreshed_field(): - return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", - "default": False, "description": "When this row was last reconciled against Odoo."} - - -def agent_fields(): - """One row per SALES AGENT, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it - is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 - carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without - the flag and 3 are flagged with no commission yet**. Either source alone silently drops real - agents. Same shape as `read_customers`' two document universes, for the same reason. - """ - return [_preset(f) for f in ( - {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " - "their commission lines instead - both are real, which is why this " - "table is the union of the two."}, - {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", - "source": "overlay", "default": True}, - # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 - # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). - # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` - # for the ruling. ⚠ This one costs more than the other two and the difference is worth - # recording: those were a click-through to a record whose id stays on the row, while this - # was an agent's BOOK — the list of customers assigned to them. The id side survives - # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in - # the data and only the rendered list is gone; the same question is answerable on the - # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on - # `sales_lines` / `sales_orders`. - _refreshed_field(), - )] - - -def account_fields(): - """One row per `account.account` — the GL chart, the owner's "GL codes". - - ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest - of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on - expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest - binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a - link into a table that does not exist. Declaring one here would render a permanently blank - column, which is the exact trap D-87 warns about from the value site. - """ - return [_preset(f) for f in ( - {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.account` id. Also this row's id."}, - # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its - # options omit is wave-26 item 24: the filter panel answers with a list that cannot match - # what is stored. - {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", - "default": True, - "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", - "income_other", "asset_cash", "asset_current", "asset_receivable", - "asset_fixed", "asset_non_current", "asset_prepayments", - "liability_current", "liability_payable", "liability_credit_card", - "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, - {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked for the expense family - the same predicate the semantic layer's " - "gl_lines topic uses, so this column and that topic cannot disagree."}, - _refreshed_field(), - )] - - -def vendor_fields(): - """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. - - ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 - vendors, of which only 9 also appear in the customer population. Pointing bills at - `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly - empty column, not an error. - """ - return [_preset(f) for f in ( - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. - # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) - # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by - # default would push the useful ones off the first screen, and DESIGN.md's "never - # over-explain" applies to columns as much as to prose. Every one is one click away in the - # field picker, and a locked database still allows fields (item 4's vocabulary). - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": True}, - {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, - {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, - {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", - "default": False}, - {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, - "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, - # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already - # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a - # shared spelling across two linked grids is how a rollup ends up summing the wrong column. - {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, - {"key": "website", "label": "Website", "type": "url", "source": "overlay", - "default": False}, - {"key": "street", "label": "Street", "type": "text", "source": "overlay", - "default": False}, - {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", - "default": False}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, - {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, - {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, - "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def bill_fields(): - """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. - - ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 - expense GL lines. What a person calls "expenses" is both, and they are different tables - the - bill is what you pay, the line is what it was coded to. This is the payable; the line ledger - is the read-through mirror grid (R2). - """ - return [_preset(f) for f in ( - {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's - # direction, so a refund reduces a total without anybody re-deriving a sign here. - {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["in_invoice", "in_refund"]}, - {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def invoice_fields(): - """One row per POSTED customer invoice or refund — the full history, not just what is open.""" - return [_preset(f) for f in ( - {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " - "why AR rollups need no filter."}, - {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the - # table held open AR only; the full posted history also carries paid / in_payment / - # reversed. A select holding a value its options do not declare is the wave-26 item-24 - # defect — the filter panel answers with a list that cannot match what is stored. - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["out_invoice", "out_refund"]}, - _scope_field(), - # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like - # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. - # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather - # than left to be inferred from a green gate. - # - # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). - # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data - # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: - # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, - # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. - # - # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: - # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` - # prefix test relaxed — which makes EVERY GATE GREEN while - # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` - # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids - # touching that line this wave for exactly that reason. - # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was - # raised from, and the mirror did not sync it until this wave — so order->invoice was a - # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. - # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold - # anything; a merged one can hold several origins space-separated). The link resolves - # against `order_no` and finds nothing when the text is not an order name — the honest - # outcome, and the reason this is a join HINT rather than a foreign key. - {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, - {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, - _refreshed_field(), - )] - - -def order_fields(): - """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, +"""odoo_relational.py — Odoo entities as LOCKED relational databases. + +Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset +Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually +replace. + +⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the +owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports +tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the +partners who owed money — so most Odoo ids were simply absent, and the missing rows were the +reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: + + ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a + posted customer document + ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code + ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund + ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order + +⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's +own `plan()` refused above it — but the cap was never a property of the store (see the measured +banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). +The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until +wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory +budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). +The four tables together are +17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split +is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); +they are answered by the read-through rollup, which never copies a row. + +⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts +GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables +dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and +carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner +holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. +Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you +filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps +excluding, because `modules/ar` is its oracle and an oracle answers ONE question. + +⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over +ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate +**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside +`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the +`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the +wider link would count 31,418 documents and call them open invoices, so those two rollups carry +the oracle's predicate as an explicit `payment_state` condition pair. +""" +import datetime as _dt + +#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: +#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. +RI_SLUGS = ("", "royal-imports") + +INVOICES_KEY = "ut_odoo_invoices" +CUSTOMERS_KEY = "ut_odoo_customers" +ORDERS_KEY = "ut_odoo_orders" +PRODUCTS_KEY = "ut_odoo_products" +#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, +#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to +#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). +AGENTS_KEY = "ut_odoo_agents" +ACCOUNTS_KEY = "ut_odoo_accounts" +BILLS_KEY = "ut_odoo_bills" +VENDORS_KEY = "ut_odoo_vendors" + +#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. +#: +#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it +#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced +#: rather than left standing beside its own contradiction. The obstacle was never the number of +#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED +#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) +#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 +#: settles what that means: *"there is no cap in how many data from the API source … can be pulled +#: into the app"*, so the answer is a different residency, never a bigger ceiling. +#: +#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the +#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this +#: database stores no rows HERE", which is a different statement from `None` ("connected and +#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator +#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work +#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with +#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working +#: grid; that is the whole shape of the conversion. +ORDER_LINES_KEY = "ut_odoo_order_lines" +GL_LINES_KEY = "ut_odoo_gl_lines" + +#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). +JOIN_KEY = "partner_id" +#: The product-grain equivalent. +PRODUCT_JOIN_KEY = "product_id" +#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but +#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the +#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. +AGENT_JOIN_KEY = "agent_id" +#: A vendor is also a `res.partner`; same reasoning, its own column. +VENDOR_JOIN_KEY = "vendor_id" +ACCOUNT_JOIN_KEY = "account_code" + +#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two +#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. +_AR_OPEN = "payment_state IN ('not_paid','partial')" +_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" +_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" +_CONFIRMED = "state IN ('sale','done')" + +#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. +#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that +#: left the population, which is right — a reversed invoice must not keep inflating a total. But +#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded +#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible +#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. +MAX_SHRINK = 0.5 + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _registry(): + """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the + route layer before `platform/` is necessarily on the path.""" + import core.registry as registry + return registry + + +def _iso_today(): + return _dt.date.today().strftime("%Y-%m-%d") + + +def _preset(field, flow="odoo_relational"): + """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the + grid renders it grey and the preset walls refuse a rename or a delete.""" + field = dict(field) + field["automation"] = {"flowId": flow, "preset": True} + return field + + +#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. +#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing +#: one here would be a second condition vocabulary beside `_clean_rollup`'s. +_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, + {"field": "payment_state", "op": "eq", "value": "partial"}], + "conditionConj": "or"} + + +# --------------------------------------------------------------------------------------------- +# FIELD CONTRACTS +# --------------------------------------------------------------------------------------------- +# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None +# for an unknown one — which DELETES the column silently on the next read rather than erroring. +def _scope_field(): + return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", + "source": "overlay", "default": False, + "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " + "metric in this product excludes. The row is kept so no Odoo id is " + "missing; filter on this column to reconcile against the AR page."} + + +def _refreshed_field(): + return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", + "default": False, "description": "When this row was last reconciled against Odoo."} + + +def agent_fields(): + """One row per SALES AGENT, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it + is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 + carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without + the flag and 3 are flagged with no commission yet**. Either source alone silently drops real + agents. Same shape as `read_customers`' two document universes, for the same reason. + """ + return [_preset(f) for f in ( + {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " + "their commission lines instead - both are real, which is why this " + "table is the union of the two."}, + {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", + "source": "overlay", "default": True}, + # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 + # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). + # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` + # for the ruling. ⚠ This one costs more than the other two and the difference is worth + # recording: those were a click-through to a record whose id stays on the row, while this + # was an agent's BOOK — the list of customers assigned to them. The id side survives + # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in + # the data and only the rendered list is gone; the same question is answerable on the + # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on + # `sales_lines` / `sales_orders`. + _refreshed_field(), + )] + + +def account_fields(): + """One row per `account.account` — the GL chart, the owner's "GL codes". + + ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest + of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on + expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest + binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a + link into a table that does not exist. Declaring one here would render a permanently blank + column, which is the exact trap D-87 warns about from the value site. + """ + return [_preset(f) for f in ( + {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.account` id. Also this row's id."}, + # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its + # options omit is wave-26 item 24: the filter panel answers with a list that cannot match + # what is stored. + {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", + "default": True, + "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", + "income_other", "asset_cash", "asset_current", "asset_receivable", + "asset_fixed", "asset_non_current", "asset_prepayments", + "liability_current", "liability_payable", "liability_credit_card", + "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, + {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked for the expense family - the same predicate the semantic layer's " + "gl_lines topic uses, so this column and that topic cannot disagree."}, + _refreshed_field(), + )] + + +def vendor_fields(): + """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. + + ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 + vendors, of which only 9 also appear in the customer population. Pointing bills at + `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly + empty column, not an error. + """ + return [_preset(f) for f in ( + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. + # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) + # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by + # default would push the useful ones off the first screen, and DESIGN.md's "never + # over-explain" applies to columns as much as to prose. Every one is one click away in the + # field picker, and a locked database still allows fields (item 4's vocabulary). + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": True}, + {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, + {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, + {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", + "default": False}, + {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, + "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, + # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already + # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a + # shared spelling across two linked grids is how a rollup ends up summing the wrong column. + {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, + {"key": "website", "label": "Website", "type": "url", "source": "overlay", + "default": False}, + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, + {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, + "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def bill_fields(): + """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. + + ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 + expense GL lines. What a person calls "expenses" is both, and they are different tables - the + bill is what you pay, the line is what it was coded to. This is the payable; the line ledger + is the read-through mirror grid (R2). + """ + return [_preset(f) for f in ( + {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's + # direction, so a refund reduces a total without anybody re-deriving a sign here. + {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["in_invoice", "in_refund"]}, + {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def invoice_fields(): + """One row per POSTED customer invoice or refund — the full history, not just what is open.""" + return [_preset(f) for f in ( + {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " + "why AR rollups need no filter."}, + {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the + # table held open AR only; the full posted history also carries paid / in_payment / + # reversed. A select holding a value its options do not declare is the wave-26 item-24 + # defect — the filter panel answers with a list that cannot match what is stored. + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["out_invoice", "out_refund"]}, + _scope_field(), + # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like + # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. + # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather + # than left to be inferred from a green gate. + # + # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). + # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data + # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: + # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, + # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. + # + # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: + # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` + # prefix test relaxed — which makes EVERY GATE GREEN while + # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` + # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids + # touching that line this wave for exactly that reason. + # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was + # raised from, and the mirror did not sync it until this wave — so order->invoice was a + # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. + # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold + # anything; a merged one can hold several origins space-separated). The link resolves + # against `order_no` and finds nothing when the text is not an order name — the honest + # outcome, and the reason this is a join HINT rather than a foreign key. + {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, + {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, + _refreshed_field(), + )] + + +def order_fields(): + """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", "default": True}, {"key": "commitment_date", "label": "Delivery date", "type": "date", @@ -407,1453 +407,1601 @@ def order_fields(): "options": ["pending", "started", "partial", "full"], "description": "Odoo's fulfilment status: pending, started, partial, or full."}, {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", - "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", - "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, - _scope_field(), - # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same - # column in `invoice_fields` above for the ruling and the refused alternative. This row - # keeps `customer` and `partner_id`, so only the click-through is gone. - # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS - # order, matched on the order's own name. - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, - _refreshed_field(), - )] - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE -# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two -# lists must name the SAME columns, and both failure directions are silent: -# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); -# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. -# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just -# add a column". -# -# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings -# rather than omissions: -# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw -# document, not the mirror), so a link at a read-through grain resolves against nothing and -# renders a permanently blank column — the trap `account_fields` names from the other end. -# The join ids ride as ordinary filterable columns instead, so the relationships are all -# still reachable by a person and by SQL. -# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by -# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column -# would be blank for every row forever. `section_line_grids` tolerates the key; the honest -# thing is not to declare it. - - -def order_line_fields(): - """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` - scope every wholesale metric in this product uses. - - MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded - sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — - but `done` stays in the option list because it is in the SCOPE, and a filter offering only - what happens to be stored today goes stale the first time an order is marked done. - - ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` - carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — - which is also what makes `order_no` a readable primary cell instead of a line id. - - ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a - FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the - type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both - render through the client's `numberText`, which rounds nothing without a `format.decimals` - bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so - it takes the type that does not paint one. - """ - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The sale order this line belongs to. Zero orders have a blank name, " - "which is why it is the primary cell rather than the line id."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order.line` id. Also this row's id."}, - {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", - "default": False, - "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 101 section and note lines, which carry no product."}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", - "default": False}, - {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, - "agg": "sum", - "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " - "(returns), so nothing here is rounded."}, - {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", - "default": False, "agg": "sum", - "description": "Odoo's own line margin. Populated on every line."}, - {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", - "default": False, - "description": "The cost Odoo priced this line's margin against, per unit."}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - _scope_field(), - )] - - -def gl_line_fields(): - """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the - grain a person can actually browse. - - MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 - draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are - invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the - reader chooses. That is the same decision the binding states from the SQL side. - - ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: - **61,911 lines carry no partner** (journal entries that are not about a customer), and - **1,727 carry no account** at all, which is also why `account_code` — the key - `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies - it is a LEFT one. - - ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: - five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · - `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. - A select whose options go stale answers a filter with a list that cannot match a stored value - (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were - measured COMPLETE against the whole table. - """ - return [_preset(f) for f in ( - {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The journal entry this line belongs to. Never blank."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move.line` id. Also this row's id."}, - {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", - "default": False}, - {"key": "account", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", - "default": True, - "description": "The GL code, from the joined chart of accounts — the key " - "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " - "account."}, - {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 61,911 lines that are not about a partner."}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, - {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, - {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", - "default": False, - "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, - {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", - "default": False}, - {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", - "default": True, "options": ["draft", "posted", "cancel"]}, - _scope_field(), - )] - - -def product_fields(): - """One row per `product.product`, keyed on its id — EVERY product, archived ones included. - - ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes - map to more than one product id (re-SKU / merge history). The code is what a human reads and - the id is what `sales_lines.product` groups by, so both are columns and only the id is the - identity. - - ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products - carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a - blank name — measured. Pinning `code` would give those rows a blank primary cell, which is - exactly D-80: a first column nothing populates quietly becoming the row's identity - ([[fallback-that-became-the-rule]]). - """ - return [_preset(f) for f in ( - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `product.product` id. Also this row's id, and what " - "every product rollup groups by."}, - {"key": "code", "label": "SKU", "type": "text", "source": "overlay", - "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " - "products that are not stocked SKUs."}, - {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Unticked = archived. Archived products are kept because they still " - "carry sales history — two of them sold this year."}, - # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a - # select whose options go stale answers a filter with a list that cannot match a stored - # value (wave-26 item 24). Text filters honestly and never goes out of date. - {"key": "category", "label": "Category", "type": "text", "source": "overlay", - "default": True}, - {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", - "default": False, "options": ["product", "consu", "service"]}, - {"key": "standard_price", "label": "Standard cost", "type": "currency", - "source": "overlay", "default": True}, - # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in - # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; - # `sales_lines` holds 256,810 rows that are never copied into this table. - {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "units", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -def customer_fields(): - """One row per partner Odoo has transacted with, keyed on the `res.partner` id. - - ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a - relation Odoo already decided. The rollups come in two kinds on purpose: - * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer - anything about a document the table holds, including a date rank; - * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query - over the whole mirror — they can answer a DATE-WINDOWED money question, which a link - rollup cannot, because a condition can only compare against a literal and a literal year - start is right until 1 January. - """ - return [_preset(f) for f in ( - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - # ⭐⭐ W37-T11 / D-165 — THE POSTAL ADDRESS, which this table declared nowhere and served - # never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own - # docstring on 2026-08-12 — *"`street`/`street2`/`zip` are not among [the mirror's 16 - # columns] … blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill - # HAS since run (`_sync_state` carries `res_partner.cols.…,street,street2,…,zip`, done - # 2026-08-15), so the half that could not be built now can be. - # ⛔ THIS IS ALSO LANE D'S DEPENDENCY. R4 puts address→coordinates in an enrichment field, - # and a geocoder pointed at a column that does not exist enriches nothing. - # ⚠ `default: False` on the postal lines, `True` on city/state — the same split - # `vendor_fields` made and for the same reason: four address columns on by default push - # the useful ones off the first screen. A locked database still allows fields, so every - # one is a click away in the picker. - {"key": "street", "label": "Street", "type": "text", "source": "overlay", - "default": False}, - {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", - "default": False}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, - {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, - {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": False}, - {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", - "default": True, - "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " - "Customers-module convention)."}, - # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this - # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are - # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to - # itself and look plausible doing it. - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - _scope_field(), - - # --- the relations ------------------------------------------------------------------- - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", - "default": True, - "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the - # agents table — zero dangling, which is why this ships as a link rather than a lookup. - {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, - - # --- link rollups over the invoice history -------------------------------------------- - # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual - # is exactly 0, so summing the full history gives the open balance to the cent. - # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. - # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while - # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / - # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two - # numbers with one name, 4x apart, in one product is how a correct figure gets reported - # as a bug; the scope belongs in the label, not only in a column somebody has to filter. - {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "description": "Open balance across every posted document, INCLUDING the Amazon " - "channel. Filter `In wholesale scope` to reconcile with the AR page.", - "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, - {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", - "source": "overlay", "default": False, "agg": "sum", - "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, - # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every - # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right - # this module refuses everywhere else. - {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", - "source": "overlay", "default": True, - "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, - # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a - # date column finds no numbers and returns BLANK — a column that renders empty forever - # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. - {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", - "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, - - # --- link rollups over the order history ---------------------------------------------- - {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "fn": "countall"}}, - {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "field": "order_date", "fn": "latest", - "sortBy": "order_date", "sortDir": "desc"}}, - - # --- source-backed (read-through) rollups --------------------------------------------- - # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted - # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced - # flag. Different grain, different question — the metric KEY carries the distinction, - # which is the whole reason a rollup may not carry SQL of its own. - {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": False, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", - "default": False, - "rollup": {"source": {"topic": "sales_orders", "measure": "orders", - "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -# --------------------------------------------------------------------------------------------- -# READING THE MIRROR -# --------------------------------------------------------------------------------------------- -def excluded_names(): - """The partner names out of wholesale scope, from the ONE place that defines them. - - Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and - the day somebody adds a channel this module would keep answering the old question. - """ - try: - import core.odoo as odoo - names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() - return {str(n).strip().lower() for n in names if str(n).strip()} - except Exception: # noqa: BLE001 - return set() - - -def excluded_ids(cur, names=None): - """The out-of-scope partner IDS, resolved against the MIRROR. - - ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. - `account_move.partner_name` is a copy taken when the document was written, and this module's - own `customers_from` says so out loud — *"a partner's name can differ across documents - (renames land on new invoices only)"*. So a rename would put some of one partner's documents - in scope and the rest out, silently, and the totals would stop reconciling with nothing to - point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. - - ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a - LIVE `search_read`, so importing it here would make spawning four locked databases fail - whenever Odoo is unreachable — including on this developer machine, where the handshake dies - on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. - - ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which - is the correct direction. 44 transacting partners carry no name at all; treating an - unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. - """ - names = names if names is not None else excluded_names() - if not names: - return set() - rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() - return {int(pid) for pid, name in rows if str(name).strip().lower() in names} - - -def columns(cur, table): - """The column names a mirror table actually has, lowercased. `set()` if the table is absent. - - ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY - phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns - added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, - `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` - keys, which `ready()` does not read. So there is a real window, right after a boot, where the - store reports READY and a column this module names does not exist yet — and DuckDB answers a - missing identifier with a Binder error, which reached the operator as a bare `500`. - ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the - whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers - degrade the COLUMN to blank and still write every id. - """ - try: - rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") - return {str(d[0]).lower() for d in rows.description} - except Exception: # noqa: BLE001 - return set() - - -def _col(have, name, default="NULL"): - """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" - return name if str(name).split(".")[-1].lower() in have else default - - -def _as_date(value): - """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the - STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" - if not value: - return "" - return str(value)[:10] - - -def _in_scope(pid, excluded): - """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" - return "" if int(pid) in excluded else "1" - - -def read_invoices(cur, excluded=None, open_only=False): - """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. - - ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the - out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and - TAGS the channel instead. Two queries would be two populations, and they drift the moment - either is edited. - - Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - where = _AR_WHERE if open_only else _POSTED_DOCS - # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in - # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does - # not have the column yet — and DuckDB answers a missing identifier with a Binder error that - # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the - # link degrades to blank for one sync cycle instead of refusing the whole spawn. - have = columns(cur, "account_move") - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " - f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " - f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r - scope = _in_scope(pid, excluded) - if open_only and not scope: - continue - out.append({ - "_id": str(mid), - "invoice_no": str(name or ""), - "odoo_id": int(mid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "invoice_date": _as_date(inv_date), - "due_date": _as_date(due), - "residual": float(residual or 0.0), - "amount_untaxed": float(untaxed or 0.0), - "payment_state": str(pay_state or ""), - "move_type": str(mtype or ""), - "origin_order": str(origin or "").strip(), - "wholesale_scope": scope, - }) - return out - - -def read_open_ar(cur, excluded=None): - """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. - - Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an - oracle answers exactly one question. It is a projection of `read_invoices`, never a second - query. - """ - return read_invoices(cur, excluded=excluded, open_only=True) - - -def read_orders(cur, excluded=None): - """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "sale_order") + "default": True, "agg": "sum"}, + {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", + "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", + "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, + _scope_field(), + # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same + # column in `invoice_fields` above for the ruling and the refused alternative. This row + # keeps `customer` and `partner_id`, so only the click-through is gone. + # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS + # order, matched on the order's own name. + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, + _refreshed_field(), + )] + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE +# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two +# lists must name the SAME columns, and both failure directions are silent: +# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); +# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. +# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just +# add a column". +# +# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings +# rather than omissions: +# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw +# document, not the mirror), so a link at a read-through grain resolves against nothing and +# renders a permanently blank column — the trap `account_fields` names from the other end. +# The join ids ride as ordinary filterable columns instead, so the relationships are all +# still reachable by a person and by SQL. +# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by +# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column +# would be blank for every row forever. `section_line_grids` tolerates the key; the honest +# thing is not to declare it. + + +def order_line_fields(): + """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` + scope every wholesale metric in this product uses. + + MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded + sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — + but `done` stays in the option list because it is in the SCOPE, and a filter offering only + what happens to be stored today goes stale the first time an order is marked done. + + ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` + carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — + which is also what makes `order_no` a readable primary cell instead of a line id. + + ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a + FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the + type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both + render through the client's `numberText`, which rounds nothing without a `format.decimals` + bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so + it takes the type that does not paint one. + """ + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The sale order this line belongs to. Zero orders have a blank name, " + "which is why it is the primary cell rather than the line id."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order.line` id. Also this row's id."}, + {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", + "default": False, + "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 101 section and note lines, which carry no product."}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", + "default": False}, + {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, + "agg": "sum", + "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " + "(returns), so nothing here is rounded."}, + {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", + "default": False, "agg": "sum", + "description": "Odoo's own line margin. Populated on every line."}, + {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", + "default": False, + "description": "The cost Odoo priced this line's margin against, per unit."}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + _scope_field(), + )] + + +def gl_line_fields(): + """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the + grain a person can actually browse. + + MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 + draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are + invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the + reader chooses. That is the same decision the binding states from the SQL side. + + ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: + **61,911 lines carry no partner** (journal entries that are not about a customer), and + **1,727 carry no account** at all, which is also why `account_code` — the key + `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies + it is a LEFT one. + + ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: + five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · + `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. + A select whose options go stale answers a filter with a list that cannot match a stored value + (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were + measured COMPLETE against the whole table. + """ + return [_preset(f) for f in ( + {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The journal entry this line belongs to. Never blank."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move.line` id. Also this row's id."}, + {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", + "default": False}, + {"key": "account", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", + "default": True, + "description": "The GL code, from the joined chart of accounts — the key " + "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " + "account."}, + {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 61,911 lines that are not about a partner."}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, + {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, + {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", + "default": False, + "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, + {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", + "default": False}, + {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", + "default": True, "options": ["draft", "posted", "cancel"]}, + _scope_field(), + )] + + +def product_fields(): + """One row per `product.product`, keyed on its id — EVERY product, archived ones included. + + ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes + map to more than one product id (re-SKU / merge history). The code is what a human reads and + the id is what `sales_lines.product` groups by, so both are columns and only the id is the + identity. + + ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products + carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a + blank name — measured. Pinning `code` would give those rows a blank primary cell, which is + exactly D-80: a first column nothing populates quietly becoming the row's identity + ([[fallback-that-became-the-rule]]). + """ + return [_preset(f) for f in ( + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `product.product` id. Also this row's id, and what " + "every product rollup groups by."}, + {"key": "code", "label": "SKU", "type": "text", "source": "overlay", + "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " + "products that are not stocked SKUs."}, + {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Unticked = archived. Archived products are kept because they still " + "carry sales history — two of them sold this year."}, + # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a + # select whose options go stale answers a filter with a list that cannot match a stored + # value (wave-26 item 24). Text filters honestly and never goes out of date. + {"key": "category", "label": "Category", "type": "text", "source": "overlay", + "default": True}, + {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", + "default": False, "options": ["product", "consu", "service"]}, + {"key": "standard_price", "label": "Standard cost", "type": "currency", + "source": "overlay", "default": True}, + # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in + # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; + # `sales_lines` holds 256,810 rows that are never copied into this table. + {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "units", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +def customer_fields(): + """One row per partner Odoo has transacted with, keyed on the `res.partner` id. + + ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a + relation Odoo already decided. The rollups come in two kinds on purpose: + * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer + anything about a document the table holds, including a date rank; + * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query + over the whole mirror — they can answer a DATE-WINDOWED money question, which a link + rollup cannot, because a condition can only compare against a literal and a literal year + start is right until 1 January. + """ + return [_preset(f) for f in ( + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + # ⭐⭐ W37-T11 / D-165 — THE POSTAL ADDRESS, which this table declared nowhere and served + # never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own + # docstring on 2026-08-12 — *"`street`/`street2`/`zip` are not among [the mirror's 16 + # columns] … blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill + # HAS since run (`_sync_state` carries `res_partner.cols.…,street,street2,…,zip`, done + # 2026-08-15), so the half that could not be built now can be. + # ⛔ THIS IS ALSO LANE D'S DEPENDENCY. R4 puts address→coordinates in an enrichment field, + # and a geocoder pointed at a column that does not exist enriches nothing. + # ⚠ `default: False` on the postal lines, `True` on city/state — the same split + # `vendor_fields` made and for the same reason: four address columns on by default push + # the useful ones off the first screen. A locked database still allows fields, so every + # one is a click away in the picker. + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, + {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": False}, + {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", + "default": True, + "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " + "Customers-module convention)."}, + # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this + # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are + # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to + # itself and look plausible doing it. + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + _scope_field(), + + # --- the relations ------------------------------------------------------------------- + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", + "default": True, + "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the + # agents table — zero dangling, which is why this ships as a link rather than a lookup. + {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, + + # --- link rollups over the invoice history -------------------------------------------- + # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual + # is exactly 0, so summing the full history gives the open balance to the cent. + # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. + # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while + # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / + # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two + # numbers with one name, 4x apart, in one product is how a correct figure gets reported + # as a bug; the scope belongs in the label, not only in a column somebody has to filter. + {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "description": "Open balance across every posted document, INCLUDING the Amazon " + "channel. Filter `In wholesale scope` to reconcile with the AR page.", + "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, + {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", + "source": "overlay", "default": False, "agg": "sum", + "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, + # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every + # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right + # this module refuses everywhere else. + {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", + "source": "overlay", "default": True, + "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, + # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a + # date column finds no numbers and returns BLANK — a column that renders empty forever + # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. + {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", + "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, + + # --- link rollups over the order history ---------------------------------------------- + {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "fn": "countall"}}, + {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "field": "order_date", "fn": "latest", + "sortBy": "order_date", "sortDir": "desc"}}, + + # --- source-backed (read-through) rollups --------------------------------------------- + # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted + # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced + # flag. Different grain, different question — the metric KEY carries the distinction, + # which is the whole reason a rollup may not carry SQL of its own. + {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": False, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", + "default": False, + "rollup": {"source": {"topic": "sales_orders", "measure": "orders", + "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# R9 TIER 1 — THE PRE-SET RELATIONS ON THE TWO REGISTRY-MODULE DATABASES (W41-T19) +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ THE CONSUMER IS `automation_engine._module_link_fields`, WHICH LOOKS THIS MODULE UP BY THE +# EXACT NAME `module_link_fields` THROUGH `getattr` (contract C8). Rename it and the feature ships +# DEAD with every gate green, which is wave 40's failure verbatim +# ([[two-lanes-one-contract-dead-feature]]). The grep that proves the wire: +# +# grep -n "module_link_fields" aios-web/api/automation_engine.py aios-web/api/odoo_relational.py +# +# ⛔ THESE DO NOT BELONG IN `platform/aios_grid_fields.json`. That file is the ODOO-SOURCED +# contract — every key in it is a column the tenant pool reads off an Odoo record — and a link +# column is neither read from Odoo nor written to it. `routes_products` already makes the same +# argument for the Image column it injects. + +#: ⛔⛔ THE JOIN COLUMN ON `customer_data` IS `pid`, **NOT** `partner_id`, AND THE OBVIOUS CHOICE +#: IS THE BROKEN ONE. `aios_grid_fields.json` calls `partner_id` *"the Odoo res.partner id — the +#: key every Odoo document joins on"*, so `from: "partner_id"` reads correct in every review — and +#: it resolves to nothing. That field is declared `"derived": true`: `modules/customer_data.pool` +#: NEVER PUTS IT ON THE ROW (measured against both of that module's two row templates, the live +#: build and the archived-ledger revival), because a customer row's `pid` IS the `res.partner` id +#: and a stored copy would be a second source for one fact. `routes_customers` mints the cell per +#: render into the `derived` channel, which the relational pass never sees: `_module_table` keys +#: its rows off the RAW pool dict. So `row.get("partner_id")` is `None`, `_join_norm` maps it to +#: `""`, and every linked cell on this grain would be blank forever with nothing red. +#: ⚠ `pid` is an int here and the `ut_odoo_*` side stores the same id as a string; `_join_norm` +#: canonicalises both to the same key, which is the case its W41 note was widened for. +#: ⚠ AND THE PRODUCT GRAIN IS THE OTHER WAY ROUND, so neither answer generalises: a product's +#: `pid` is a CRC32 of its SKU and the Odoo id cannot be recovered from it, which is exactly why +#: W41-T17 landed `product_id` there as a genuine pool column (`"source": "odoo"`, NOT derived). +#: Whichever grain a future link is written for, read that grain's own contract first. +CUSTOMER_JOIN_FROM = "pid" + +#: ⛔ THE AGENT RELATION JOINS ON THE NAME, AND IT IS A JOIN **HINT** RATHER THAN A FOREIGN KEY — +#: the same honesty `invoice_fields.order_link` already declares about `invoice_origin`. +#: +#: Odoo's own model, read from `ir.model.fields` rather than assumed: `res.partner.agent_ids` is a +#: **many2many back onto `res.partner`**, stored, labelled "Agents". So the relation is real and +#: Odoo declares it — but it lives in a link TABLE, not in a column, and `customer_data` mirrors +#: only the resolved display name (`agent`). There is no agent id anywhere on this grain. +#: ⚠ `agent_fields`' own note claims *"`agent_id` on `customer_data`"*. That is STALE at HEAD: +#: `customer_data` carries `agent` and nothing else about the agent, and adding an id column means +#: editing the canonical contract, which is not this ticket's fence. +#: ⛔⛔ AND IT IS STILL NOT `JOIN_KEY`. An agent IS a `res.partner`, so joining these two tables +#: through the partner id would link EVERY CUSTOMER TO ITSELF and look entirely plausible doing +#: it. The name is a weaker key than an id and a wrong key is not a key at all. +AGENT_NAME_KEY = "agent" + + +def module_link_fields(key): + """The PRE-SET relation columns on ONE registry-module database, or `[]` when it declares none. + + ⭐⭐ R9 TIER 1: *relations Odoo's own model already implies*, shipped as LINK FIELDS rather + than as a `relations:` registry (R9 refuses the registry). `automation_engine._module_table` + appends what this returns to the module's field contract, and `relation_cells` resolves it. + + ⛔ WHAT IS DELIBERATELY NOT HERE, AND EACH ABSENCE IS A FINDING RATHER THAN AN OMISSION — the + posture `account_fields` and the two read-through grains already take in this file: + + * **NO ROLLUPS.** The retired `ut_odoo_customers` spec folded `residual` / `amount_untaxed` / + `countall` / `due_date` over its `invoices` link, and copying those onto `customer_data` + would COLLIDE: `modules/customer_data.pool` already emits `ar_outstanding` and `last_order` + as reconciled, wholesale-scoped cells under those exact keys. A link rollup over + `ut_odoo_invoices` answers the ALL-CHANNELS question (the Amazon partner alone is 75% of the + raw balance), so the same key would carry two numbers 4x apart depending on which pass ran + last. That is the second-source-for-one-fact defect this module refuses everywhere else. + * **NO LINK ON `product_data` AT ALL** — see `_PRODUCT_LINK_BLOCKER` below for what it would + take. Declaring one anyway is the worst available outcome: `ut_odoo_order_lines` EXISTS in + the store with zero rows, so `_resolve_table` finds it, nothing is refused, no + `limit_report` is raised, and the column renders permanently blank while looking configured. + + ⚠ THE FIELD BAG IS SHAPED FOR THE RENDER WIRE THAT DOES NOT EXIST YET. `routes_customers` / + `routes_products` still build their field lists from `aios_grid` alone, so today these columns + live only inside the relational pass. `source: "odoo"` + `derived: True` is the shape + `partner_id` already uses, and it is what `aios_grid.rows_from_pool` needs: it SKIPS a derived + field when projecting the pool row and fills the cell from the `derived` channel instead. + A bag with no `source` key at all would raise there (`field["source"]`, not `.get`). + """ + if str(key or "") != "customer_data": + return [] + return [_preset(f) for f in ( + # Odoo declares this one: `res.partner.invoice_ids`, one2many to `account.move` with + # `relation_field = partner_id` — so the join column is Odoo's own answer, not our guess. + {"key": "invoices", "label": "Invoices", "type": "link", "source": "odoo", + "derived": True, "default": True, + "description": "Every posted invoice and refund for this customer, across all channels.", + "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": CUSTOMER_JOIN_FROM}}, + # `res.partner.sale_order_ids`, one2many to `sale.order`, `relation_field = partner_id`. + {"key": "orders", "label": "Orders", "type": "link", "source": "odoo", + "derived": True, "default": True, + "description": "Every confirmed sale order for this customer.", + "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": CUSTOMER_JOIN_FROM}}, + # ⚠ NOT `single`. Odoo's `agent_ids` is a many2many and `modules/customers` joins the + # names with ", " when a customer carries more than one, so "a customer has one agent" is + # not true of the data. Capping the cell at one would hide the second agent silently; + # letting the join miss a multi-agent row is visible and is reported instead. + {"key": "agent_link", "label": "Agent record", "type": "link", "source": "odoo", + "derived": True, "default": False, + "description": "The agent record for this account, matched on the agent name. Blank " + "when the account has no agent or carries more than one.", + "link": {"table": AGENTS_KEY, "on": AGENT_NAME_KEY, "from": AGENT_NAME_KEY}}, + )] + + +#: ⛔⛔ "PRODUCTS ORDERED" / "CUSTOMERS WHO ORDERED" CANNOT BE DECLARED HERE YET, AND THE REASON IS +#: NOT A MISSING DECLARATION — IT IS A MISSING PRIMITIVE. Recorded as data so the next lane reads +#: a measurement rather than an empty function. +#: +#: WHAT ODOO SAYS (read from `ir.model.fields`, not assumed): `product.product` declares NO +#: relation to `res.partner` at all — its only partner-shaped field is `seller_ids`, an unstored +#: one2many to `product.supplierinfo`, which is VENDORS. The one declared path between a customer +#: and a product is `sale.order.line`, which carries `order_partner_id` (many2one to `res.partner`) +#: and `product_id` (many2one to `product.product`), both stored. So the relation is real, Odoo +#: does not declare it directly, and it is exactly TWO HOPS. +#: +#: WHY NEITHER HOP IS EXPRESSIBLE TODAY: +#: 1. `relation_cells` resolves a link with ONE index lookup — `_linked_rows_by_join(target, on)` +#: probed by the source row's single `from` value. There is no `through` leg in the bag +#: vocabulary (`core.user_tables._clean_link` accepts table/on/from/inverse/reciprocal/single) +#: and no two-hop branch in the engine. +#: 2. The bridge table has no rows to walk. `ut_odoo_order_lines` is a READ-THROUGH grain by +#: design (254,189 lines / 63.9 MB) — `row_limit` answers 0 and `plan()` builds zero rows — +#: so a link pointed at it resolves against an empty index. Worse than an error: the table +#: EXISTS, so nothing is refused and no `limit_report` is emitted. +#: 3. `inverse` cannot carry the other end either. It reads the source row's STORED cell, and a +#: registry module stores no rows here: its grid is assembled per render from the tenant pool, +#: so the reciprocal cell a pool row would need is precisely the projection being computed. +#: +#: WHAT IT WOULD TAKE (three parts, one ticket, and none of them in this file): +#: (a) a `through` leg on the link bag, resolved as a two-hop join in `automation_engine`; +#: (b) bridge rows served from the DuckDB MIRROR rather than the `user_tables` document, because +#: order lines are read-through on purpose and materialising them is the thing R6 forbade; +#: (c) a cap-and-report story for the fan-out under standing rule 1 — the widest customer's +#: basket and the widest product's buyer list both need a display cap plus a +#: `limit_report`-shaped account of it, never a silent truncation. +_PRODUCT_LINK_BLOCKER = { + "subject": "product_data / customer_data: Products ordered and Customers who ordered", + "effect": "unresolvable", + "cause": "the only relation Odoo declares between a partner and a product runs through " + "sale.order.line (order_partner_id then product_id), which is two hops; the " + "relation engine resolves one hop, and the bridge grain ut_odoo_order_lines is " + "read-through by design and stores no rows to walk", + "recommendation": "add a `through` leg to the link bag resolved in automation_engine, backed " + "by the DuckDB mirror rather than the user_tables document, with a display " + "cap that is reported rather than applied silently", +} + + +# --------------------------------------------------------------------------------------------- +# READING THE MIRROR +# --------------------------------------------------------------------------------------------- +def excluded_names(): + """The partner names out of wholesale scope, from the ONE place that defines them. + + Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and + the day somebody adds a channel this module would keep answering the old question. + """ + try: + import core.odoo as odoo + names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() + return {str(n).strip().lower() for n in names if str(n).strip()} + except Exception: # noqa: BLE001 + return set() + + +def excluded_ids(cur, names=None): + """The out-of-scope partner IDS, resolved against the MIRROR. + + ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. + `account_move.partner_name` is a copy taken when the document was written, and this module's + own `customers_from` says so out loud — *"a partner's name can differ across documents + (renames land on new invoices only)"*. So a rename would put some of one partner's documents + in scope and the rest out, silently, and the totals would stop reconciling with nothing to + point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. + + ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a + LIVE `search_read`, so importing it here would make spawning four locked databases fail + whenever Odoo is unreachable — including on this developer machine, where the handshake dies + on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. + + ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which + is the correct direction. 44 transacting partners carry no name at all; treating an + unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. + """ + names = names if names is not None else excluded_names() + if not names: + return set() + rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() + return {int(pid) for pid, name in rows if str(name).strip().lower() in names} + + +def columns(cur, table): + """The column names a mirror table actually has, lowercased. `set()` if the table is absent. + + ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY + phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns + added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, + `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` + keys, which `ready()` does not read. So there is a real window, right after a boot, where the + store reports READY and a column this module names does not exist yet — and DuckDB answers a + missing identifier with a Binder error, which reached the operator as a bare `500`. + ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the + whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers + degrade the COLUMN to blank and still write every id. + """ + try: + rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") + return {str(d[0]).lower() for d in rows.description} + except Exception: # noqa: BLE001 + return set() + + +def _col(have, name, default="NULL"): + """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" + return name if str(name).split(".")[-1].lower() in have else default + + +def _as_date(value): + """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the + STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" + if not value: + return "" + return str(value)[:10] + + +def _in_scope(pid, excluded): + """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" + return "" if int(pid) in excluded else "1" + + +def read_invoices(cur, excluded=None, open_only=False): + """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. + + ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the + out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and + TAGS the channel instead. Two queries would be two populations, and they drift the moment + either is edited. + + Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + where = _AR_WHERE if open_only else _POSTED_DOCS + # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in + # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does + # not have the column yet — and DuckDB answers a missing identifier with a Binder error that + # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the + # link degrades to blank for one sync cycle instead of refusing the whole spawn. + have = columns(cur, "account_move") + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " + f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " + f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r + scope = _in_scope(pid, excluded) + if open_only and not scope: + continue + out.append({ + "_id": str(mid), + "invoice_no": str(name or ""), + "odoo_id": int(mid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "invoice_date": _as_date(inv_date), + "due_date": _as_date(due), + "residual": float(residual or 0.0), + "amount_untaxed": float(untaxed or 0.0), + "payment_state": str(pay_state or ""), + "move_type": str(mtype or ""), + "origin_order": str(origin or "").strip(), + "wholesale_scope": scope, + }) + return out + + +def read_open_ar(cur, excluded=None): + """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. + + Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an + oracle answers exactly one question. It is a projection of `read_invoices`, never a second + query. + """ + return read_invoices(cur, excluded=excluded, open_only=True) + + +def read_orders(cur, excluded=None): + """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "sale_order") sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " f" state, amount_untaxed, {_col(have, 'invoice_status')}, " f" {_col(have, 'commitment_date')}, {_col(have, 'delivery_status')} " f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): + out = [] + for r in cur.execute(sql).fetchall(): (oid, name, when, pid, pname, team, state, untaxed, inv_status, commitment, delivery) = r - out.append({ - "_id": str(oid), - "order_no": str(name or ""), - "odoo_id": int(oid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "order_date": _as_date(when), - "amount_untaxed": float(untaxed or 0.0), - "team": str(team or ""), + out.append({ + "_id": str(oid), + "order_no": str(name or ""), + "odoo_id": int(oid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "order_date": _as_date(when), + "amount_untaxed": float(untaxed or 0.0), + "team": str(team or ""), "state": str(state or ""), "invoice_status": str(inv_status or ""), "commitment_date": _as_date(commitment), "delivery_status": str(delivery or ""), "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_products(cur): - """[(row dict)] — EVERY `product.product`, keyed on its id. - - ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they - were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products - that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three - uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product - grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to - land — silently. 119 extra rows is the whole cost of the claim being literally true. - """ - have = columns(cur, "product_product") - sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " - f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " - "FROM product_product") - out = [] - for r in cur.execute(sql).fetchall(): - (prid, code, name, categ, ptype, cost, active) = r - out.append({ - "_id": str(prid), - "product": str(name or ""), - PRODUCT_JOIN_KEY: int(prid), - "code": str(code or ""), - "active": "1" if active else "", - "category": str(categ or ""), - "product_type": str(ptype or ""), - "standard_price": float(cost or 0.0), - }) - return out - - -def read_customers(cur, excluded=None): - """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. - - The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale - order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. - - ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — - the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old - docstring said partners with no document are *"left out on purpose — a row that can never - appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same - join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** - (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer - a salesperson has not sold to yet is exactly the row a prospecting view needs. - - ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the - document legs for the rank leg would drop **16 partners that hold posted documents** (7 - archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry - `partner_id` LINKS straight back here — so those links would dangle with nothing reporting - it. When other tables point AT a population, a widening must be a SUPERSET. - - ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same - `columns()`/`_col` discipline every other optional column here uses — but note the difference - honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the - POPULATION back to the document union. It degrades to today's behaviour rather than to an - empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the - column is missing so the narrowing can never pass for done. - - ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above - states the rule for this module and it applies with more force to a POPULATION than to a name - list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its - mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this - time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the - weather. One source, always present at spawn time: the mirror. - - ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR - partners; sourcing a customer registry from its own receivables is what kept most Odoo ids - out of the store in the first place. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "res_partner") - # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join - # itself names the column, so `_col` on the SELECT list alone would still fail to bind. - agent = ("ag.name" if "agent_id" in have else "NULL") - agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") - join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") - # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an - # archived prospect out, and a rank test without it would re-admit the 47 archived partners - # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. - rank_leg = (" OR (p.customer_rank > 0 AND p.active) " - if {"customer_rank", "active"} <= have else "") - # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like - # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL - # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the - # live 500 that rule was written after). - sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " - f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, " - f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} " - "FROM res_partner p " - f"{join}" - "WHERE p.id IN (" - f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " - " UNION " - " SELECT partner_id FROM account_move " - f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" - f"{rank_leg}") - out = [] - for r in cur.execute(sql).fetchall(): - (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r - out.append({ - "_id": str(pid), - "customer": str(name or ""), - JOIN_KEY: int(pid), - "street": str(street or ""), - "street2": str(street2 or ""), - "city": str(city or ""), - "state": str(state or ""), - "zip": str(zipc or ""), - "country": str(country or ""), - "agent": str(agent or ""), - AGENT_JOIN_KEY: int(agent_id) if agent_id else "", - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_agents(cur): - """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. - - ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: - Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list - silently becomes NULL and every row would read "not an agent" indistinguishably from - "unknown". Read it as a truth value, never as a presence test. - """ - have = columns(cur, "res_partner") - if "id" not in have: - return [] - flagged = "p.agent" if "agent" in have else "FALSE" - # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that - # distinction is the whole point of this block. `columns()` was written for a missing COLUMN - # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that - # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent - # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table - # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is - # precisely D-107's shape, and it would have arrived on the first deploy of this feature. - # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the - # commission table the population falls back to the FLAGGED partners alone and `commissioned` - # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. - has_comm = bool(columns(cur, "account_invoice_line_agent")) - commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " - " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") - union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " - " UNION " if has_comm else "") - sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " - "FROM res_partner p WHERE p.id IN (" - f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") - out = [] - for (aid, name, flag, comm) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(aid), - "agent": str(name or ""), - "odoo_id": int(aid), - AGENT_JOIN_KEY: int(aid), - "flagged": "1" if flag else "", - "commissioned": "1" if comm else "", - }) - return out - - -def read_accounts(cur): - """[(row dict)] — the whole GL chart, keyed on the `account.account` id. - - ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: - `harness/semantic.py`'s `gl_lines` topic scopes expenses as - `account_type in ('expense','expense_depreciation')`. A second definition here is how a - column and a topic start disagreeing about the same word. - ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so - there is nothing to filter and every account is a row. - """ - have = columns(cur, "account_account") - if not have: - return [] - sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " - f" {_col(have, 'name', chr(39) + chr(39))}, " - f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") - out = [] - for (aid, code, name, atype) in cur.execute(sql).fetchall(): - t = str(atype or "") - out.append({ - "_id": str(aid), - ACCOUNT_JOIN_KEY: str(code or ""), - "account_name": str(name or ""), - "odoo_id": int(aid), - "account_type": t, - "is_expense": "1" if t in ("expense", "expense_depreciation") else "", - }) - return out - - -_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" - - -def read_bills(cur): - """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" - have = columns(cur, "account_move") - if not have: - return [] - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - f" {_col(have, 'amount_untaxed_signed', '0')}, " - f" {_col(have, 'amount_residual_signed', '0')}, " - f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " - f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r - out.append({ - "_id": str(mid), - "bill_no": str(name or ""), - "odoo_id": int(mid), - "vendor": str(pname or ""), - VENDOR_JOIN_KEY: int(pid), - "invoice_date": _as_date(when), - "due_date": _as_date(due), - "amount_untaxed": float(untaxed or 0.0), - "residual": float(residual or 0.0), - "payment_state": str(pay or ""), - "move_type": str(mtype or ""), - }) - return out - - -def read_vendors(cur): - """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. - - ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its - invoices. The asymmetry is intentional and the reason is what that function's own comment - says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. - There is no second document universe for vendors — a partner with no bill has no payable - history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills - carry a null partner; all 393 vendors resolve in `res_partner`). - """ - have = columns(cur, "res_partner") - if not have or not columns(cur, "account_move"): - return [] - # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured - # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a - # vendor record for — how to reach them, who they are for tax, and where they are. - # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the - # column. That is not defensive habit: this projection has to keep working against a mirror - # that has not been re-synced since the widening, and the alternative is a reader that raises - # on the exact box the widening was meant to help. The columns arrive when the backfill runs; - # until then these read blank rather than failing. - blank = chr(39) + chr(39) - sql = ("SELECT p.id, p.name, " - f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " - f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " - f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " - f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " - f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " - f"{_col(have, 'p.zip', blank)} " - "FROM res_partner p WHERE p.id IN " - f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " - " AND partner_id IS NOT NULL)") - out = [] - for (pid, name, country, email, phone, mobile, website, vat, ref, - street, street2, city, zipc) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(pid), - "vendor": str(name or ""), - "odoo_id": int(pid), - VENDOR_JOIN_KEY: int(pid), - "country": str(country or ""), - "email": str(email or ""), - "phone": str(phone or ""), - "mobile": str(mobile or ""), - "website": str(website or ""), - "vat": str(vat or ""), - "vendor_ref": str(ref or ""), - "street": str(street or ""), - "street2": str(street2 or ""), - "city": str(city or ""), - "zip": str(zipc or ""), - }) - return out - - -def customers_from(invoice_rows): - """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. - - ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a - pure function over rows and the gate uses it to prove the FOLD against a fixture without a - mirror; deleting it would cost a test its independence from the SQL. - """ - out = {} - for row in invoice_rows: - pid = row[JOIN_KEY] - entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], - JOIN_KEY: pid}) - # A partner's name can differ across documents (renames land on new invoices only); - # the newest non-empty one wins so the locked table shows what Odoo shows today. - if row["customer"]: - entry["customer"] = row["customer"] - return list(out.values()) - - -# --------------------------------------------------------------------------------------------- -# THE SPAWN -# --------------------------------------------------------------------------------------------- -class Refused(Exception): - """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have - been written wrong.""" - - -#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole -#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. -#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. -TABLES = ( - # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / - # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules - # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current - # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it - # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries - # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). - # Dropping the row here is what stops them being planned, built or re-created; - # `RETIRED_KEYS` below is what removes the rows a tenant already has. - ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), - ("orders", ORDERS_KEY, "Odoo orders", order_fields), - # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders - # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" - # is four more spec rows and four readers rather than a new substrate. - ("agents", AGENTS_KEY, "Odoo agents", agent_fields), - ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), - ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), - ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), - # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that - # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) - # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of - # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` - # does not name, so the grids would be bound to the mirror and unreachable, which is this - # wave's own [[reachable-is-not-the-same-as-built]] shape. - ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), - ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), -) - -#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` -#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. -#: -#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: -#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so -#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that -#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the -#: key, and this is the shape that says so. -#: -#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all -#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the -#: database, never over which Odoo model was queried. Same for `account_move`, which is the -#: customer-invoice book under one WHERE and the vendor-bill book under another. -TABLE_SUBJECTS = { - # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects - # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` - # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one - # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future - # spec row tried to bring it back. - INVOICES_KEY: "odoo:account.move.customer", - ORDERS_KEY: "odoo:sale.order", - AGENTS_KEY: "odoo:res.partner.agent", - ACCOUNTS_KEY: "odoo:account.account", - VENDORS_KEY: "odoo:res.partner.vendor", - BILLS_KEY: "odoo:account.move.vendor", - ORDER_LINES_KEY: "odoo:sale.order.line", - GL_LINES_KEY: "odoo:account.move.line", -} - -#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. -#: -#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the -#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the -#: owner actually sees in the nav. This set is what removes them, and it is applied inside -#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every -#: resync. -#: -#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a -#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms -#: it — and the running Space reverts it within a minute, because the container holds the document -#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a -#: script is a dry run that reports success. -#: -#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job -#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This -#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, -#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes -#: and re-adds the table every 30 minutes. -RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) - -#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. -#: -#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness -#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was -#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote -#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new -#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned -#: the gate RED until this constant went with them. It did, and that is the ratchet working. -#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. - -#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no -#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a -#: policy `plan()` could forget. The two line grains are absent for that reason and no other. -_READERS = { - "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), - "products": lambda cur, excluded: read_products(cur), - "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), - "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), - "agents": lambda cur, excluded: read_agents(cur), - "accounts": lambda cur, excluded: read_accounts(cur), - "vendors": lambda cur, excluded: read_vendors(cur), - "bills": lambda cur, excluded: read_bills(cur), -} - -#: The table keys this module can never materialise — DERIVED from the absence of a reader, never -#: typed out, so it cannot drift from the fact it describes. -#: -#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE -#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a -#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a -#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps -#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early -#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is -#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the -#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. -#: The conversion writes the stamp; a table that needs no conversion still needs the statement. -READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) - - -class _LentDoc: - """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. - - ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` - resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE - 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per - table per loop, so passing the live handle would have added ~16 full document copies to a - function that already reads it exactly once — and with `st=None` (the gate's fixture posture, - and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset - fetch per table, on a path that has no business touching the network at all. - `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` - takes `st` rather than `defn`, so the lending happens one level up, here. - - ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim - that quietly proxied other keys would be a second store with a partial view, which is worse - than one that says what it knows. - """ - - def __init__(self, doc, key): - self._doc, self._key = doc if isinstance(doc, dict) else {}, key - - def get(self, name): - return self._doc if name == self._key else None - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner opened the Odoo connector and found nothing to configure: no key, no server database, -# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all -# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey -# it — a config the route knows and the sync path does not is a switch that flips nothing. -# -#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` -CONFIG_KEY = "odoo_connector_config" - -#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The -#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a -#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. -#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None -#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather -#: than a silent once-a-century schedule. -SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} -SYNC_FLOOR_SECONDS = 1800 -DEFAULT_SYNC = "30m" - - -def read_config(rt): - """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not - make a connector look disconnected.""" - try: - cur = rt.get(CONFIG_KEY) if rt is not None else None - except Exception: # noqa: BLE001 - cur = None - cur = cur if isinstance(cur, dict) else {} - grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} - every = cur.get("syncEvery") - return {"grids": {str(k): bool(v) for k, v in grids.items()}, - "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, - "frozen": bool(cur.get("frozen")), - "frozenAt": str(cur.get("frozenAt") or "")} - - -def grid_choices(rt): - """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from - `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). - - ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is - what they have today; only an explicit untick turns one off. The alternative — an empty - config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. - """ - chosen = read_config(rt)["grids"] - return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} - for _bucket, key, label, _fields in TABLES] - - -def enabled_buckets(rt): - """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" - chosen = read_config(rt)["grids"] - return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} - - -def sync_seconds(rt): - """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). - - ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that - predates the preset list, or one written by any path that is not the route, must still not be - able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that - holds until somebody finds the second way in [[limit-with-no-enforcer]]. - """ - secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) - if secs is None: - return None - return max(int(secs), SYNC_FLOOR_SECONDS) - - -def frozen(rt): - """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and - stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" - return bool(read_config(rt)["frozen"]) - - -def plan(cur, rt=None): - """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. - - Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and - so **every cap is checked before anything is committed**. - - Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty - list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — - `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's - second sentence carried as data rather than left for a reader to infer from an empty list. - - ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn - bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would - create some and refuse the rest — leaving a locked invoices database with no rollup host, - while the route answered as though nothing had happened. A partial spawn is worse than a - refused one, so the count is checked against the tables that ALREADY exist, before the first - write. `rt` also carries the stored row counts the shrink guard compares against. - """ - ut = _ut() - # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from - # three places in falling authority: the registry, a stored `connected: True`, then the - # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a - # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one - # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently - # not apply on exactly the run that creates the databases. This module DECLARES these keys, so - # it is the honest place to say what they are. Idempotent (a set add), and it fills the - # evaluator's input rather than becoming a second evaluator. - ut.register_connected(*[key for _b, key, _l, _f in TABLES]) - excluded = excluded_ids(cur) - # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is - # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = - # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this - # function as the caller that reads it, which is the seam working as designed — one evaluator, - # so the spawn, the write doors and the wire cannot disagree about whether a table is capped - # ([[one-evaluator-per-question]]). - # - # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: - # * no reader at all — structural, permanent, and the case that must not depend on a store - # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); - # * a reader exists but the table has already been converted to read-through — the - # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them - # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` - # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table - # D-87's conversion had already emptied. - # - # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used - # to be read after the build loop; it moved up because `row_limit` needs it and reading it per - # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face - # fetch per table on a path that must never touch the network). - existing = {} - if rt is not None: - try: - existing = dict(rt.get(ut.STORE_KEY) or {}) - except Exception: # noqa: BLE001 - existing = {} - # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and - # without it R6's report is silently absent on the run that matters most — the FIRST spawn. - # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so - # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and - # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids - # were skipped with no stated reason. The structural `reader is None` guard still did its job; - # what went missing was the half of R6 that has to SAY WHY. - # - # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` - # would work in one line, and `core.user_tables` explicitly reserves that registrar — - # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for - # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold - # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could - # be eligible FOR), but the law is written without an exception, so this lends the evaluator - # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so - # what is lent is the document as it stands the moment this plan applies. - lent = _LentDoc({**existing, - **{k: {**(existing.get(k) or {}), "readThrough": True} - for k in READ_THROUGH_KEYS}}, - ut.STORE_KEY) - - built, limits = {}, {} - caps = {} - for bucket, key, _label, _fields in TABLES: - reader = _READERS.get(bucket) - caps[key] = cap = ut.row_limit(key, st=lent) - if reader is None or cap == 0: - built[bucket] = [] - report = ut.limit_report(key, st=lent) - if report: - limits[key] = report - continue - built[bucket] = reader(cur, excluded) - problems = [] - - if rt is not None: - needed = [k for _b, k, _l, _f in TABLES if k not in existing] - if needed and len(existing) + len(needed) > ut.MAX_TABLES: - problems.append( - f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " - f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " - f"part of a linked set") - - for bucket, key, _label, _fields in TABLES: - rows = built[bucket] - cap = caps[key] # asked ONCE per table, above — never re-read per loop - # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED - # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in - # how many data from the API source (as long as its from a connected source like Odoo) that - # can be pulled into the app… Now if there is lag or it can't be done, you need to - # explicitly tell me why and recommend a fix."* Both halves are here: a connected table - # answers `None` and is never refused for its size, and anything that IS still bounded is - # reported with its cause and its recommendation instead of a hand-typed line. - # - # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: - # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and - # the loop above already handed it an empty list. Neither may reach the refusal. The - # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped - # table understates every total it feeds while looking exactly like a complete one. - if cap and len(rows) > cap: - report = ut.limit_report(key, st=lent) or {} - limits[key] = report - problems.append( - f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " - f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " - f"{report.get('recommendation', '')}".strip()) - # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every - # spawn after the first conversion: zero rows against a stored population is the INTENDED - # end state there, not the partial mirror read this guard exists to catch. - if cap == 0: - continue - stored = len(((existing.get(key) or {}).get("rows")) or {}) - if stored and len(rows) < stored * MAX_SHRINK: - problems.append( - f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " - f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " - f"shrinking; refusing rather than deleting rows that still exist") - built["problems"] = problems - # R6's second sentence as DATA rather than prose: every table whose rows this plan did not - # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. - # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that - # is not a bucket name is inert there, exactly as `problems` has always been. - built["limits"] = limits - return built - - -def apply_plan(rt, built, username="automation", today=None, report=None): - """Create-or-merge every table in `built` and its rows. Idempotent by construction. - - Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the - same record — which is also what makes "every Odoo unique id is in the database" a checkable - statement rather than a hopeful one. - - ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. - """ - if built.get("problems"): - raise Refused("; ".join(built["problems"])) - stamp = today or _iso_today() - written = {} - # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and - # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. - retired = [] - plans = [(key, label, fields(), built[bucket]) - for bucket, key, label, fields in TABLES if bucket in built] - - # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. - # - # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full - # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB - # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits - # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). - # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. - # - # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at - # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had - # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to - # create. Inside one updater, a raise aborts before anything is persisted. - def _apply_all(cur): - cur = cur if isinstance(cur, dict) else {} - # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE - # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every - # resync), which is the only place a change to the tenant document survives (D-195: the - # same edit from a CLI reports success and is reverted within a minute). - # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot - # contain them, and there is no door here that could re-add one. `removed` is reported so - # a caller can SAY what happened rather than infer it from a table going missing. - # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them - # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its - # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die - # with `TypeError: list indices must be integers`. One dict, two value shapes, is a - # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through - # its OWN channel and the return type stays exactly what it was. - for key in RETIRED_KEYS: - if cur.pop(key, None) is not None: - retired.append(key) - for key, label, fields, rows in plans: - written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _apply_all, flush="sync") - if report is not None and retired: - report["retired"] = sorted(retired) - return written - - -def _ensure_table(rt, key, label, fields, rows, username, stamp): - """One table, written on its own. Kept because the gate drives a single table directly, and - because a caller with one table to reconcile should not have to compose an updater.""" - written = {} - - def _one(cur): - cur = cur if isinstance(cur, dict) else {} - written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _one, flush="sync") - return written["counts"] - - -def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): - """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. - - ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — - but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` - was built FROM open invoices, removal meant a customer who paid their bill vanished from the - registry; now a partner leaves only when their last document does. The shrink guard in - `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. - """ - ut = _ut() - wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} - for row in wanted.values(): - row["refreshed"] = stamp - counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} - subject = TABLE_SUBJECTS.get(key) - table = cur.get(key) - if table is None: - if len(cur) >= ut.MAX_TABLES: - # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a - # successful refresh over a table that does not exist. - raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") - # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. - # - # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This - # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key - # on every pass. A claim test outside this `if` refuses the table it created last boot, - # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would - # take the product down to prevent a duplicate that already exists. Refusing the SECOND - # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to - # remove, and it is removed by dropping its `TABLES` row, not by a guard here. - # - # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. - # Never a module-level cache: one Space process serves every tenant, and two tenants both - # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated - # here). A stored definition with no `subject` claims nothing. - held = {t["subject"]: k for k, t in cur.items() - if isinstance(t, dict) and t.get("subject")} - # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered - # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. - other = _registry().subject_conflict(subject, key, claimed=held) - if other: - raise Refused( - f"{key}: refusing to create a second database for {subject!r} — {other!r} " - f"already presents it. One subject, one database (item 12a); if this table is " - f"meant to replace {other!r}, retire {other!r} first rather than shipping both") - table = cur[key] = { - "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, - "createdBy": username, "created": stamp, "fields": [], "rows": {}, - # recordMode = a LOCKED database (item-3 nomenclature): no human may add or - # delete records, while fields stay addable. Odoo owns this population. - "recordMode": ut.AUTOMATION_RECORD_MODE, - } - table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) - # W30-T35 — the durable "my rows are not in this document" statement, on the tables no - # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not - # `setdefault`: it is derived from the code's own structure, so the code is what it must agree - # with, and a definition that somehow lost the flag should regain it rather than keep serving - # an empty grid. - if key in READ_THROUGH_KEYS: - table["readThrough"] = True - # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same - # reason `readThrough` is: it is derived from this module's own structure, so the code is what - # it has to agree with, and a definition that lost the stamp should regain it rather than go - # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees - # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold - # process reading a fresh document has nothing else to read the claim off. - if subject: - table["subject"] = subject - have = {str(f.get("key")): f for f in (table.get("fields") or [])} - for field in fields: - # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. - # - # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND - # patch, so every field that goes through the normal door has one. This function does NOT - # go through that door — it writes definitions straight into the document — and - # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that - # ever omitted it would 500 the entire rows route rather than degrade one column. Measured - # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is - # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that - # prompted it came from a hand-written test fixture, not from production). - # - # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a - # missing `source` means the definition does not say which stratum owns the column, and - # rendering it as blank would bury that. The default matches what the validator would - # have stamped, so the bypass stops being a hole without inventing a second rule. - if isinstance(field, dict) and not field.get("source"): - field = {**field, "source": "overlay"} - fkey = str(field.get("key")) - if fkey not in have: - table.setdefault("fields", []).append(dict(field)) - continue - # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The - # widening moved `payment_state`'s option list and every rollup's conditions; a - # create-only merge would have left the LIVE table declaring the old contract - # forever, so the column would render but its filter could not match what is stored. - # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a - # column a user added to a locked database is never rewritten. - stored = have[fkey] - # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, - # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the - # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on - # an Odoo database would silently revert at the next boot rebuild. - if _ut().user_edited(stored): - continue - if (stored.get("automation") or {}).get("preset"): - for prop in ("label", "type", "options", "link", "rollup", "description", - "agg", "pinned", "default"): - if prop in field: - stored[prop] = field[prop] - else: - stored.pop(prop, None) - stored_rows = table.setdefault("rows", {}) - for rid, values in wanted.items(): - current = stored_rows.get(rid) - if current is None: - stored_rows[rid] = dict(values) - counts["added"] += 1 - elif any(str(current.get(k, "")) != str(v) for k, v in values.items() - if k != "refreshed"): - current.update(values) - counts["updated"] += 1 - else: - current["refreshed"] = values["refreshed"] - for rid in [r for r in stored_rows if r not in wanted]: - stored_rows.pop(rid, None) - counts["removed"] += 1 - return counts - - -def is_royal(tenant): - return str(tenant or "").strip().lower() in RI_SLUGS - - -def refresh(rt, tenant, username="automation", cur=None, today=None): - """THE entry point — the store-resync path and the route both call this. - - ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row - still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than - silently authoritative. - """ - if not is_royal(tenant): - raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " - f"Imports only); refusing to spawn empty locked databases") - # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, - # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is - # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process - # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a - # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked - # databases from another customer's rows — a spawn, not a read, so the wrong numbers become - # durable. Entitlement is not residency. - # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot - # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. - if rt is not None: - rt.assert_datastore_matches() - # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. - # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing - # the credential alone is not a freeze — this function is also reached by the boot rebuild and - # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a - # disconnected workspace would silently re-materialise from `.env` on the next tick and the - # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: - # the resync loop calling this every cycle must not be handed an exception as a status. - if rt is not None and frozen(rt): - return {"tables": {}, "frozen": True, - "note": "this workspace has disconnected Odoo; its databases are frozen as " - "static data and are not being refreshed"} - if cur is None: - from harness import datastore - cur = datastore.ro_con() - built = plan(cur, rt=rt) - # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the - # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a - # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so - # every cap, refusal and limit report is still computed over the full set — a config must not - # be able to hide a problem by hiding the table that has it. - # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from - # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not - # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. - if rt is not None: - keep = enabled_buckets(rt) - skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) - for bucket, _key, _l, _f in TABLES: - if bucket not in keep: - built.pop(bucket, None) - else: - skipped = [] - # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. - # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no - # longer built) SAYS SO, with the keys. - plan_report = {} - written = apply_plan(rt, built, username=username, today=today, report=plan_report) - return {"tables": written, - **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), - # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. - **({"skipped": skipped} if skipped else {}), - **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} + }) + return out + + +def read_products(cur): + """[(row dict)] — EVERY `product.product`, keyed on its id. + + ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they + were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products + that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three + uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product + grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to + land — silently. 119 extra rows is the whole cost of the claim being literally true. + """ + have = columns(cur, "product_product") + sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " + f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " + "FROM product_product") + out = [] + for r in cur.execute(sql).fetchall(): + (prid, code, name, categ, ptype, cost, active) = r + out.append({ + "_id": str(prid), + "product": str(name or ""), + PRODUCT_JOIN_KEY: int(prid), + "code": str(code or ""), + "active": "1" if active else "", + "category": str(categ or ""), + "product_type": str(ptype or ""), + "standard_price": float(cost or 0.0), + }) + return out + + +def read_customers(cur, excluded=None): + """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. + + The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale + order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. + + ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — + the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old + docstring said partners with no document are *"left out on purpose — a row that can never + appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same + join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** + (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer + a salesperson has not sold to yet is exactly the row a prospecting view needs. + + ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the + document legs for the rank leg would drop **16 partners that hold posted documents** (7 + archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry + `partner_id` LINKS straight back here — so those links would dangle with nothing reporting + it. When other tables point AT a population, a widening must be a SUPERSET. + + ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same + `columns()`/`_col` discipline every other optional column here uses — but note the difference + honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the + POPULATION back to the document union. It degrades to today's behaviour rather than to an + empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the + column is missing so the narrowing can never pass for done. + + ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above + states the rule for this module and it applies with more force to a POPULATION than to a name + list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its + mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this + time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the + weather. One source, always present at spawn time: the mirror. + + ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR + partners; sourcing a customer registry from its own receivables is what kept most Odoo ids + out of the store in the first place. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "res_partner") + # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join + # itself names the column, so `_col` on the SELECT list alone would still fail to bind. + agent = ("ag.name" if "agent_id" in have else "NULL") + agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") + join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") + # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an + # archived prospect out, and a rank test without it would re-admit the 47 archived partners + # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. + rank_leg = (" OR (p.customer_rank > 0 AND p.active) " + if {"customer_rank", "active"} <= have else "") + # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like + # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL + # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the + # live 500 that rule was written after). + sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " + f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, " + f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} " + "FROM res_partner p " + f"{join}" + "WHERE p.id IN (" + f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " + " UNION " + " SELECT partner_id FROM account_move " + f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" + f"{rank_leg}") + out = [] + for r in cur.execute(sql).fetchall(): + (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r + out.append({ + "_id": str(pid), + "customer": str(name or ""), + JOIN_KEY: int(pid), + "street": str(street or ""), + "street2": str(street2 or ""), + "city": str(city or ""), + "state": str(state or ""), + "zip": str(zipc or ""), + "country": str(country or ""), + "agent": str(agent or ""), + AGENT_JOIN_KEY: int(agent_id) if agent_id else "", + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_agents(cur): + """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. + + ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: + Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list + silently becomes NULL and every row would read "not an agent" indistinguishably from + "unknown". Read it as a truth value, never as a presence test. + """ + have = columns(cur, "res_partner") + if "id" not in have: + return [] + flagged = "p.agent" if "agent" in have else "FALSE" + # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that + # distinction is the whole point of this block. `columns()` was written for a missing COLUMN + # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that + # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent + # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table + # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is + # precisely D-107's shape, and it would have arrived on the first deploy of this feature. + # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the + # commission table the population falls back to the FLAGGED partners alone and `commissioned` + # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. + has_comm = bool(columns(cur, "account_invoice_line_agent")) + commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " + " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") + union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " + " UNION " if has_comm else "") + sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " + "FROM res_partner p WHERE p.id IN (" + f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") + out = [] + for (aid, name, flag, comm) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(aid), + "agent": str(name or ""), + "odoo_id": int(aid), + AGENT_JOIN_KEY: int(aid), + "flagged": "1" if flag else "", + "commissioned": "1" if comm else "", + }) + return out + + +def read_accounts(cur): + """[(row dict)] — the whole GL chart, keyed on the `account.account` id. + + ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: + `harness/semantic.py`'s `gl_lines` topic scopes expenses as + `account_type in ('expense','expense_depreciation')`. A second definition here is how a + column and a topic start disagreeing about the same word. + ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so + there is nothing to filter and every account is a row. + """ + have = columns(cur, "account_account") + if not have: + return [] + sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " + f" {_col(have, 'name', chr(39) + chr(39))}, " + f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") + out = [] + for (aid, code, name, atype) in cur.execute(sql).fetchall(): + t = str(atype or "") + out.append({ + "_id": str(aid), + ACCOUNT_JOIN_KEY: str(code or ""), + "account_name": str(name or ""), + "odoo_id": int(aid), + "account_type": t, + "is_expense": "1" if t in ("expense", "expense_depreciation") else "", + }) + return out + + +_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" + + +def read_bills(cur): + """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" + have = columns(cur, "account_move") + if not have: + return [] + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + f" {_col(have, 'amount_untaxed_signed', '0')}, " + f" {_col(have, 'amount_residual_signed', '0')}, " + f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " + f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r + out.append({ + "_id": str(mid), + "bill_no": str(name or ""), + "odoo_id": int(mid), + "vendor": str(pname or ""), + VENDOR_JOIN_KEY: int(pid), + "invoice_date": _as_date(when), + "due_date": _as_date(due), + "amount_untaxed": float(untaxed or 0.0), + "residual": float(residual or 0.0), + "payment_state": str(pay or ""), + "move_type": str(mtype or ""), + }) + return out + + +def read_vendors(cur): + """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. + + ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its + invoices. The asymmetry is intentional and the reason is what that function's own comment + says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. + There is no second document universe for vendors — a partner with no bill has no payable + history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills + carry a null partner; all 393 vendors resolve in `res_partner`). + """ + have = columns(cur, "res_partner") + if not have or not columns(cur, "account_move"): + return [] + # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured + # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a + # vendor record for — how to reach them, who they are for tax, and where they are. + # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the + # column. That is not defensive habit: this projection has to keep working against a mirror + # that has not been re-synced since the widening, and the alternative is a reader that raises + # on the exact box the widening was meant to help. The columns arrive when the backfill runs; + # until then these read blank rather than failing. + blank = chr(39) + chr(39) + sql = ("SELECT p.id, p.name, " + f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " + f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " + f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " + f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " + f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " + f"{_col(have, 'p.zip', blank)} " + "FROM res_partner p WHERE p.id IN " + f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " + " AND partner_id IS NOT NULL)") + out = [] + for (pid, name, country, email, phone, mobile, website, vat, ref, + street, street2, city, zipc) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(pid), + "vendor": str(name or ""), + "odoo_id": int(pid), + VENDOR_JOIN_KEY: int(pid), + "country": str(country or ""), + "email": str(email or ""), + "phone": str(phone or ""), + "mobile": str(mobile or ""), + "website": str(website or ""), + "vat": str(vat or ""), + "vendor_ref": str(ref or ""), + "street": str(street or ""), + "street2": str(street2 or ""), + "city": str(city or ""), + "zip": str(zipc or ""), + }) + return out + + +def customers_from(invoice_rows): + """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. + + ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a + pure function over rows and the gate uses it to prove the FOLD against a fixture without a + mirror; deleting it would cost a test its independence from the SQL. + """ + out = {} + for row in invoice_rows: + pid = row[JOIN_KEY] + entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], + JOIN_KEY: pid}) + # A partner's name can differ across documents (renames land on new invoices only); + # the newest non-empty one wins so the locked table shows what Odoo shows today. + if row["customer"]: + entry["customer"] = row["customer"] + return list(out.values()) + + +# --------------------------------------------------------------------------------------------- +# THE SPAWN +# --------------------------------------------------------------------------------------------- +class Refused(Exception): + """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have + been written wrong.""" + + +#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole +#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. +#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. +TABLES = ( + # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / + # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules + # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current + # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it + # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries + # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). + # Dropping the row here is what stops them being planned, built or re-created; + # `RETIRED_KEYS` below is what removes the rows a tenant already has. + ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), + ("orders", ORDERS_KEY, "Odoo orders", order_fields), + # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders + # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" + # is four more spec rows and four readers rather than a new substrate. + ("agents", AGENTS_KEY, "Odoo agents", agent_fields), + ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), + ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), + ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), + # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that + # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) + # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of + # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` + # does not name, so the grids would be bound to the mirror and unreachable, which is this + # wave's own [[reachable-is-not-the-same-as-built]] shape. + ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), + ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), +) + +#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` +#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. +#: +#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: +#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so +#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that +#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the +#: key, and this is the shape that says so. +#: +#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all +#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the +#: database, never over which Odoo model was queried. Same for `account_move`, which is the +#: customer-invoice book under one WHERE and the vendor-bill book under another. +TABLE_SUBJECTS = { + # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects + # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` + # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one + # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future + # spec row tried to bring it back. + INVOICES_KEY: "odoo:account.move.customer", + ORDERS_KEY: "odoo:sale.order", + AGENTS_KEY: "odoo:res.partner.agent", + ACCOUNTS_KEY: "odoo:account.account", + VENDORS_KEY: "odoo:res.partner.vendor", + BILLS_KEY: "odoo:account.move.vendor", + ORDER_LINES_KEY: "odoo:sale.order.line", + GL_LINES_KEY: "odoo:account.move.line", +} + +#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. +#: +#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the +#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the +#: owner actually sees in the nav. This set is what removes them, and it is applied inside +#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every +#: resync. +#: +#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a +#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms +#: it — and the running Space reverts it within a minute, because the container holds the document +#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a +#: script is a dry run that reports success. +#: +#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job +#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This +#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, +#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes +#: and re-adds the table every 30 minutes. +RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) + +#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. +#: +#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness +#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was +#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote +#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new +#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned +#: the gate RED until this constant went with them. It did, and that is the ratchet working. +#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. + +#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no +#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a +#: policy `plan()` could forget. The two line grains are absent for that reason and no other. +_READERS = { + "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), + "products": lambda cur, excluded: read_products(cur), + "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), + "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), + "agents": lambda cur, excluded: read_agents(cur), + "accounts": lambda cur, excluded: read_accounts(cur), + "vendors": lambda cur, excluded: read_vendors(cur), + "bills": lambda cur, excluded: read_bills(cur), +} + +#: The table keys this module can never materialise — DERIVED from the absence of a reader, never +#: typed out, so it cannot drift from the fact it describes. +#: +#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE +#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a +#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a +#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps +#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early +#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is +#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the +#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. +#: The conversion writes the stamp; a table that needs no conversion still needs the statement. +READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) + + +class _LentDoc: + """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. + + ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` + resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE + 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per + table per loop, so passing the live handle would have added ~16 full document copies to a + function that already reads it exactly once — and with `st=None` (the gate's fixture posture, + and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset + fetch per table, on a path that has no business touching the network at all. + `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` + takes `st` rather than `defn`, so the lending happens one level up, here. + + ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim + that quietly proxied other keys would be a second store with a partial view, which is worse + than one that says what it knows. + """ + + def __init__(self, doc, key): + self._doc, self._key = doc if isinstance(doc, dict) else {}, key + + def get(self, name): + return self._doc if name == self._key else None + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner opened the Odoo connector and found nothing to configure: no key, no server database, +# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all +# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey +# it — a config the route knows and the sync path does not is a switch that flips nothing. +# +#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` +CONFIG_KEY = "odoo_connector_config" + +#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The +#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a +#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. +#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None +#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather +#: than a silent once-a-century schedule. +SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} +SYNC_FLOOR_SECONDS = 1800 +DEFAULT_SYNC = "30m" + + +def read_config(rt): + """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not + make a connector look disconnected.""" + try: + cur = rt.get(CONFIG_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + cur = None + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} + every = cur.get("syncEvery") + return {"grids": {str(k): bool(v) for k, v in grids.items()}, + "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, + "frozen": bool(cur.get("frozen")), + "frozenAt": str(cur.get("frozenAt") or "")} + + +def grid_choices(rt): + """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from + `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). + + ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is + what they have today; only an explicit untick turns one off. The alternative — an empty + config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. + """ + chosen = read_config(rt)["grids"] + return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} + for _bucket, key, label, _fields in TABLES] + + +def enabled_buckets(rt): + """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" + chosen = read_config(rt)["grids"] + return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} + + +def sync_seconds(rt): + """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). + + ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that + predates the preset list, or one written by any path that is not the route, must still not be + able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that + holds until somebody finds the second way in [[limit-with-no-enforcer]]. + """ + secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) + if secs is None: + return None + return max(int(secs), SYNC_FLOOR_SECONDS) + + +def frozen(rt): + """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and + stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" + return bool(read_config(rt)["frozen"]) + + +def plan(cur, rt=None): + """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. + + Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and + so **every cap is checked before anything is committed**. + + Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty + list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — + `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's + second sentence carried as data rather than left for a reader to infer from an empty list. + + ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn + bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would + create some and refuse the rest — leaving a locked invoices database with no rollup host, + while the route answered as though nothing had happened. A partial spawn is worse than a + refused one, so the count is checked against the tables that ALREADY exist, before the first + write. `rt` also carries the stored row counts the shrink guard compares against. + """ + ut = _ut() + # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from + # three places in falling authority: the registry, a stored `connected: True`, then the + # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a + # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one + # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently + # not apply on exactly the run that creates the databases. This module DECLARES these keys, so + # it is the honest place to say what they are. Idempotent (a set add), and it fills the + # evaluator's input rather than becoming a second evaluator. + ut.register_connected(*[key for _b, key, _l, _f in TABLES]) + excluded = excluded_ids(cur) + # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is + # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = + # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this + # function as the caller that reads it, which is the seam working as designed — one evaluator, + # so the spawn, the write doors and the wire cannot disagree about whether a table is capped + # ([[one-evaluator-per-question]]). + # + # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: + # * no reader at all — structural, permanent, and the case that must not depend on a store + # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); + # * a reader exists but the table has already been converted to read-through — the + # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them + # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` + # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table + # D-87's conversion had already emptied. + # + # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used + # to be read after the build loop; it moved up because `row_limit` needs it and reading it per + # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face + # fetch per table on a path that must never touch the network). + existing = {} + if rt is not None: + try: + existing = dict(rt.get(ut.STORE_KEY) or {}) + except Exception: # noqa: BLE001 + existing = {} + # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and + # without it R6's report is silently absent on the run that matters most — the FIRST spawn. + # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so + # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and + # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids + # were skipped with no stated reason. The structural `reader is None` guard still did its job; + # what went missing was the half of R6 that has to SAY WHY. + # + # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` + # would work in one line, and `core.user_tables` explicitly reserves that registrar — + # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for + # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold + # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could + # be eligible FOR), but the law is written without an exception, so this lends the evaluator + # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so + # what is lent is the document as it stands the moment this plan applies. + lent = _LentDoc({**existing, + **{k: {**(existing.get(k) or {}), "readThrough": True} + for k in READ_THROUGH_KEYS}}, + ut.STORE_KEY) + + built, limits = {}, {} + caps = {} + for bucket, key, _label, _fields in TABLES: + reader = _READERS.get(bucket) + caps[key] = cap = ut.row_limit(key, st=lent) + if reader is None or cap == 0: + built[bucket] = [] + report = ut.limit_report(key, st=lent) + if report: + limits[key] = report + continue + built[bucket] = reader(cur, excluded) + problems = [] + + if rt is not None: + needed = [k for _b, k, _l, _f in TABLES if k not in existing] + if needed and len(existing) + len(needed) > ut.MAX_TABLES: + problems.append( + f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " + f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " + f"part of a linked set") + + for bucket, key, _label, _fields in TABLES: + rows = built[bucket] + cap = caps[key] # asked ONCE per table, above — never re-read per loop + # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED + # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in + # how many data from the API source (as long as its from a connected source like Odoo) that + # can be pulled into the app… Now if there is lag or it can't be done, you need to + # explicitly tell me why and recommend a fix."* Both halves are here: a connected table + # answers `None` and is never refused for its size, and anything that IS still bounded is + # reported with its cause and its recommendation instead of a hand-typed line. + # + # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: + # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and + # the loop above already handed it an empty list. Neither may reach the refusal. The + # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped + # table understates every total it feeds while looking exactly like a complete one. + if cap and len(rows) > cap: + report = ut.limit_report(key, st=lent) or {} + limits[key] = report + problems.append( + f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " + f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " + f"{report.get('recommendation', '')}".strip()) + # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every + # spawn after the first conversion: zero rows against a stored population is the INTENDED + # end state there, not the partial mirror read this guard exists to catch. + if cap == 0: + continue + stored = len(((existing.get(key) or {}).get("rows")) or {}) + if stored and len(rows) < stored * MAX_SHRINK: + problems.append( + f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " + f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " + f"shrinking; refusing rather than deleting rows that still exist") + built["problems"] = problems + # R6's second sentence as DATA rather than prose: every table whose rows this plan did not + # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. + # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that + # is not a bucket name is inert there, exactly as `problems` has always been. + built["limits"] = limits + return built + + +def apply_plan(rt, built, username="automation", today=None, report=None): + """Create-or-merge every table in `built` and its rows. Idempotent by construction. + + Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the + same record — which is also what makes "every Odoo unique id is in the database" a checkable + statement rather than a hopeful one. + + ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. + """ + if built.get("problems"): + raise Refused("; ".join(built["problems"])) + stamp = today or _iso_today() + written = {} + # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and + # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. + retired = [] + plans = [(key, label, fields(), built[bucket]) + for bucket, key, label, fields in TABLES if bucket in built] + + # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. + # + # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full + # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB + # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits + # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). + # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. + # + # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at + # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had + # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to + # create. Inside one updater, a raise aborts before anything is persisted. + def _apply_all(cur): + cur = cur if isinstance(cur, dict) else {} + # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE + # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every + # resync), which is the only place a change to the tenant document survives (D-195: the + # same edit from a CLI reports success and is reverted within a minute). + # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot + # contain them, and there is no door here that could re-add one. `removed` is reported so + # a caller can SAY what happened rather than infer it from a table going missing. + # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them + # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its + # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die + # with `TypeError: list indices must be integers`. One dict, two value shapes, is a + # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through + # its OWN channel and the return type stays exactly what it was. + for key in RETIRED_KEYS: + if cur.pop(key, None) is not None: + retired.append(key) + for key, label, fields, rows in plans: + written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _apply_all, flush="sync") + if report is not None and retired: + report["retired"] = sorted(retired) + return written + + +def _ensure_table(rt, key, label, fields, rows, username, stamp): + """One table, written on its own. Kept because the gate drives a single table directly, and + because a caller with one table to reconcile should not have to compose an updater.""" + written = {} + + def _one(cur): + cur = cur if isinstance(cur, dict) else {} + written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _one, flush="sync") + return written["counts"] + + +def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): + """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. + + ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — + but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` + was built FROM open invoices, removal meant a customer who paid their bill vanished from the + registry; now a partner leaves only when their last document does. The shrink guard in + `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. + """ + ut = _ut() + wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} + for row in wanted.values(): + row["refreshed"] = stamp + counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} + subject = TABLE_SUBJECTS.get(key) + table = cur.get(key) + if table is None: + if len(cur) >= ut.MAX_TABLES: + # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a + # successful refresh over a table that does not exist. + raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") + # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. + # + # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This + # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key + # on every pass. A claim test outside this `if` refuses the table it created last boot, + # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would + # take the product down to prevent a duplicate that already exists. Refusing the SECOND + # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to + # remove, and it is removed by dropping its `TABLES` row, not by a guard here. + # + # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. + # Never a module-level cache: one Space process serves every tenant, and two tenants both + # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated + # here). A stored definition with no `subject` claims nothing. + held = {t["subject"]: k for k, t in cur.items() + if isinstance(t, dict) and t.get("subject")} + # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered + # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. + other = _registry().subject_conflict(subject, key, claimed=held) + if other: + raise Refused( + f"{key}: refusing to create a second database for {subject!r} — {other!r} " + f"already presents it. One subject, one database (item 12a); if this table is " + f"meant to replace {other!r}, retire {other!r} first rather than shipping both") + table = cur[key] = { + "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, + "createdBy": username, "created": stamp, "fields": [], "rows": {}, + # recordMode = a LOCKED database (item-3 nomenclature): no human may add or + # delete records, while fields stay addable. Odoo owns this population. + "recordMode": ut.AUTOMATION_RECORD_MODE, + } + table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) + # W30-T35 — the durable "my rows are not in this document" statement, on the tables no + # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not + # `setdefault`: it is derived from the code's own structure, so the code is what it must agree + # with, and a definition that somehow lost the flag should regain it rather than keep serving + # an empty grid. + if key in READ_THROUGH_KEYS: + table["readThrough"] = True + # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same + # reason `readThrough` is: it is derived from this module's own structure, so the code is what + # it has to agree with, and a definition that lost the stamp should regain it rather than go + # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees + # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold + # process reading a fresh document has nothing else to read the claim off. + if subject: + table["subject"] = subject + have = {str(f.get("key")): f for f in (table.get("fields") or [])} + for field in fields: + # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. + # + # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND + # patch, so every field that goes through the normal door has one. This function does NOT + # go through that door — it writes definitions straight into the document — and + # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that + # ever omitted it would 500 the entire rows route rather than degrade one column. Measured + # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is + # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that + # prompted it came from a hand-written test fixture, not from production). + # + # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a + # missing `source` means the definition does not say which stratum owns the column, and + # rendering it as blank would bury that. The default matches what the validator would + # have stamped, so the bypass stops being a hole without inventing a second rule. + if isinstance(field, dict) and not field.get("source"): + field = {**field, "source": "overlay"} + fkey = str(field.get("key")) + if fkey not in have: + table.setdefault("fields", []).append(dict(field)) + continue + # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The + # widening moved `payment_state`'s option list and every rollup's conditions; a + # create-only merge would have left the LIVE table declaring the old contract + # forever, so the column would render but its filter could not match what is stored. + # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a + # column a user added to a locked database is never rewritten. + stored = have[fkey] + # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, + # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the + # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on + # an Odoo database would silently revert at the next boot rebuild. + if _ut().user_edited(stored): + continue + if (stored.get("automation") or {}).get("preset"): + for prop in ("label", "type", "options", "link", "rollup", "description", + "agg", "pinned", "default"): + if prop in field: + stored[prop] = field[prop] + else: + stored.pop(prop, None) + stored_rows = table.setdefault("rows", {}) + for rid, values in wanted.items(): + current = stored_rows.get(rid) + if current is None: + stored_rows[rid] = dict(values) + counts["added"] += 1 + elif any(str(current.get(k, "")) != str(v) for k, v in values.items() + if k != "refreshed"): + current.update(values) + counts["updated"] += 1 + else: + current["refreshed"] = values["refreshed"] + for rid in [r for r in stored_rows if r not in wanted]: + stored_rows.pop(rid, None) + counts["removed"] += 1 + return counts + + +def is_royal(tenant): + return str(tenant or "").strip().lower() in RI_SLUGS + + +def refresh(rt, tenant, username="automation", cur=None, today=None): + """THE entry point — the store-resync path and the route both call this. + + ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row + still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than + silently authoritative. + """ + if not is_royal(tenant): + raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " + f"Imports only); refusing to spawn empty locked databases") + # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, + # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is + # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process + # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a + # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked + # databases from another customer's rows — a spawn, not a read, so the wrong numbers become + # durable. Entitlement is not residency. + # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot + # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. + if rt is not None: + rt.assert_datastore_matches() + # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. + # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing + # the credential alone is not a freeze — this function is also reached by the boot rebuild and + # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a + # disconnected workspace would silently re-materialise from `.env` on the next tick and the + # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: + # the resync loop calling this every cycle must not be handed an exception as a status. + if rt is not None and frozen(rt): + return {"tables": {}, "frozen": True, + "note": "this workspace has disconnected Odoo; its databases are frozen as " + "static data and are not being refreshed"} + if cur is None: + from harness import datastore + cur = datastore.ro_con() + built = plan(cur, rt=rt) + # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the + # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a + # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so + # every cap, refusal and limit report is still computed over the full set — a config must not + # be able to hide a problem by hiding the table that has it. + # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from + # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not + # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. + if rt is not None: + keep = enabled_buckets(rt) + skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) + for bucket, _key, _l, _f in TABLES: + if bucket not in keep: + built.pop(bucket, None) + else: + skipped = [] + # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. + # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no + # longer built) SAYS SO, with the keys. + plan_report = {} + written = apply_plan(rt, built, username=username, today=today, report=plan_report) + return {"tables": written, + **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), + # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. + **({"skipped": skipped} if skipped else {}), + **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} diff --git a/api/providers.py b/api/providers.py index 5425c862136396406714fecaa23eec0d3288ec52..d294400d33431e3b8c3bda9253357544cf451875 100644 --- a/api/providers.py +++ b/api/providers.py @@ -1,897 +1,897 @@ -"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. - -⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if -anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize -when we scale to thousands+ customer … make sure our code is customizable and modular."* - -⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". -A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious -shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments -correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A -provider-level fallback would notice the missing views and re-buy the profile, the likes and the -comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that -is noise; at thousands of customers × twelve posts each it is the whole bill. - -So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: - - ig_profile -> brightdata (36 fields, works) - ig_post_metrics -> brightdata (likes + comments, works) - ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) - ig_comments -> brightdata - -Each capability has an ORDERED list of providers. The first one that is configured AND capable AND -succeeds wins; the rest are never called, so the common path costs exactly one vendor record. - -⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in -the one column you asked for has failed for our purposes, and a chain that only catches exceptions -would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying -answer exactly as it would on a 500. - -⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time -(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what -lets a tenant be moved off a vendor without a release. -""" -from __future__ import annotations - -import json -import os -import time -from dataclasses import dataclass, field -from typing import Callable - -# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in -# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the -# line that stops a container booting. That asymmetry is the whole reason the two are imported -# differently — see `anthropic_sdk()`. -import requests - -#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK -#: providers and to estimate a run's spend for the operator. They are not billing truth — the -#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is -#: why they are named and dated rather than guessed silently. -#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper -#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. -#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number -#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before -#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. -#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here -#: reads a live price — a modelled figure that says so is honest; one that pretends is not. -_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) -_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) - - -@dataclass(frozen=True) -class Capability: - """What ONE provider can do for ONE capability, and what it costs to ask.""" - #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never - #: called for this capability at all — it cannot be reached by a fallback, cannot be put first - #: by an env override, and cannot silently start being billed because somebody reordered a - #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route - #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four - #: accounts from 13K to 268M followers returned an account-grain constant or a null. - capable: bool = True - cost_per_record: float = 0.0 - #: Free-text, shown to an operator deciding where their money went. - note: str = "" - - -@dataclass -class Provider: - key: str - label: str - #: The env var holding this provider's credential. NEVER the credential itself — this module - #: is imported by surfaces that serialise their config. - key_env: str = "" - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True - - def cap(self, capability: str) -> Capability | None: - return self.caps.get(capability) - - def can(self, capability: str) -> bool: - c = self.cap(capability) - return bool(c and c.capable and self.configured()) - - -#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the -#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. -PROVIDERS: dict[str, Provider] = { - "brightdata": Provider( - key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", - caps={ - "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), - "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), - "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), - # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. - "ig_post_views": Capability( - False, _COST_BRIGHTDATA, - "MEASURED INCAPABLE: returns one account-grain number for every reel of a " - "creator (identical for two shortcodes in one call, ticking upward between " - "calls) and never populates video_play_count. Their own documented example " - "account no longer reproduces it."), - # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our - # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own - # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. - "tt_profile": Capability(True, _COST_BRIGHTDATA, - "40 fields incl. bio/engagement rates/region"), - # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. - # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says - # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post - # record and a separate view rung would be a second bill for a number we already have. - # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: - # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an - # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it - # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a - # quiet fallback bolted onto this one. - "tt_post_metrics": Capability( - True, _COST_BRIGHTDATA, - "likes + comments + shares + saves, and play_count inline (DECLARED " - "no-empties-or-zeros, UNPROVEN until one live pull)"), - "tt_comments": Capability(True, _COST_BRIGHTDATA, - "separate paid dataset, opt-in: 17 fields"), - }), - "apify": Provider( - key="apify", label="Apify", key_env="AIOS_APIFY_KEY", - caps={ - # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and - # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. - "ig_post_views": Capability(True, _COST_APIFY, - "videoPlayCount, matches Instagram's displayed views"), - "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), - "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), - }), -} - -#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. -DEFAULT_CHAINS: dict[str, tuple] = { - "ig_profile": ("brightdata", "apify"), - "ig_post_metrics": ("brightdata", "apify"), - # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here - # would be a lie that costs a wasted call on every single post. - "ig_post_views": ("apify",), - "ig_comments": ("brightdata",), - # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. - # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a - # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data - # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap - # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here - # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, - # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; - # it is not declared until a runner walks it. - "tt_profile": ("brightdata",), - "tt_post_metrics": ("brightdata",), - "tt_comments": ("brightdata",), -} - - -def chain(capability: str) -> list: - """The provider order for `capability` — env override first, then the default. - - `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the - owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a - vendor that is having a bad day. - ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — - `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore - resolves to EMPTY rather than to a provider that would return a wrong number, because a - plausible wrong number is worse than an honest refusal. - """ - raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() - names = None - for clause in raw.split(";"): - if "=" in clause: - cap_name, _, order = clause.partition("=") - if cap_name.strip() == capability: - names = [x.strip() for x in order.split(",") if x.strip()] - if names is None: - names = list(DEFAULT_CHAINS.get(capability) or ()) - return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] - - -def estimate(capability: str, records: int) -> dict: - """What the FIRST capable provider would cost for `records` — the number an operator plans on. - - Reported per capability rather than per run because that is the unit that scales: at a thousand - tenants the question is never "what did this run cost", it is "what does adding view counts to - every post cost per month". - """ - ch = chain(capability) - if not ch: - return {"capability": capability, "records": records, "provider": None, "usd": 0.0, - "note": "no configured, capable provider"} - p = ch[0] - c = p.cap(capability) - return {"capability": capability, "records": records, "provider": p.key, - "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), - "note": c.note} - - -@dataclass -class Attempt: - provider: str - ok: bool - note: str = "" - records: int = 0 - seconds: float = 0.0 - - -def run(capability, work, satisfied=None, log=None): - """Walk the chain for `capability` until one provider gives a SATISFYING answer. - - `work(provider) -> (result, note)`; a non-empty note means it did not answer. - `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any - truthy result. - - Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it - satisfied, and how long it took — because "where did this number come from and what did it - cost" is a question somebody asks about a bill, not about a stack trace. - - ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the - failure it must catch is not an error: a vendor answering 200 with the one field you needed - left blank. That is exactly how Bright Data behaves on view counts, and a chain that only - caught exceptions would have stopped there forever and never reached Apify. - """ - log = log or (lambda *_a: None) - ok = satisfied or (lambda r: bool(r)) - attempts, last = [], None - for provider in chain(capability): - started = time.time() - try: - result, note = work(provider) - except Exception as e: # noqa: BLE001 - # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is - # one refactor away from being a query param; this line must not be what leaks it. - result, note = None, f"{type(e).__name__}" - secs = round(time.time() - started, 2) - if note: - attempts.append(Attempt(provider.key, False, note, 0, secs)) - log(f" {provider.label}: {note}, falling through") - continue - if not ok(result): - attempts.append(Attempt(provider.key, False, "answered without the field asked for", - 0, secs)) - log(f" {provider.label}: answered, but not with what was asked for, falling through") - last = result if last is None else last - continue - attempts.append(Attempt(provider.key, True, "", _count(result), secs)) - log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") - return result, attempts - return last, attempts - - -def _count(result): - if isinstance(result, (list, tuple, set)): - return len(result) - if isinstance(result, dict): - return len(result) - return 1 if result else 0 - - -def wire(): - """What the Settings surface renders. Booleans and labels — NEVER a credential. - - Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a - provider is usable, never what the key is. - """ - return { - "providers": [ - {"key": p.key, "label": p.label, "configured": p.configured(), - "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} - for p in PROVIDERS.values() - ], - "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, - "incapable": { - f"{p.key}:{cap}": c.note - for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable - }, - } - - - - -# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ -# -# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the -# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: -# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this -# shape'."* -# -# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the -# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is -# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a -# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. -# -# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is -# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is -# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half -# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the -# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both -# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. -# -# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that -# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. -# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling -# turn at all, so the guess never has to be made. - -#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: -#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen -#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short -#: enough that a top-up is picked up without a restart. -CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) - -#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a -#: latency optimisation, not a billing record. A second container learns the same thing from its -#: own first 402, and neither one can be wrong for longer than the cooldown. -_NO_CREDIT: dict[str, float] = {} - - -@dataclass(frozen=True) -class LlmProvider: - """One chat-completions endpoint, and what it is DECLARED able to do.""" - name: str - label: str - env: str - url: str - model: str - #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, - #: which is a different body, a different auth header and a different result shape. - wire: str - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.env) or "").strip()) - - def can(self, capability: str) -> bool: - cap = self.caps.get(capability) - return bool(cap and cap.capable and self.configured()) - - -#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put -#: cerebras first *"because this path needs tool calling and cerebras carries this account's -#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always -#: works, and the others are the cheap seats it falls through to. -LLM_PROVIDERS: dict[str, LlmProvider] = { - "anthropic": LlmProvider( - name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", - url="https://api.anthropic.com/v1/messages", - # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with - # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier - # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. - # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real - # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. - model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", - wire="anthropic", - caps={ - # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a - # typed `tool_use` content block carrying parsed `input`. There is nothing to recover - # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to - # apologise for on the other wire. - "llm_tool_calling": Capability(True, 0.0, - "typed tool_use content block; no text recovery path"), - "llm_chat": Capability(True, 0.0, "Messages API"), - "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), - }), - "cerebras": LlmProvider( - name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", - url="https://api.cerebras.ai/v1/chat/completions", - # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the - # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on - # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` - # already knows how to survive (R4: skipped, not tried). - model="gpt-oss-120b", wire="openai", - caps={ - # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not - # measure this rung: a 402 comes back before any tool is considered, so there is no - # observation to record. Leaving the old wording is the honest answer — upgrading it - # to MEASURED beside its two neighbours would be claiming an account balance as - # evidence about a capability. - "llm_tool_calling": Capability(True, 0.0, - "DECLARED, not measured: this account's tool-capable " - "model per the wave-32 ladder note. ⚠ 2026-08-19: the " - "account answers 402, so this stays unmeasured"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "groq": LlmProvider( - name="groq", label="Groq", env="GROQ_API_KEY", - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq - # RETIRED it. Measured, not inferred — a real POST with the live key answered - # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with - # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A - # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the - # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). - # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. - # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while - # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY - # spelled differently and a sweep that "normalises" them breaks one of them. - url="https://api.groq.com/openai/v1/chat/completions", - model="openai/gpt-oss-120b", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old - # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME - # provider on this wire answers 400 with the call in `failed_generation`; the repo - # never recorded which. Flip this to False the day it is"*. It is recorded now: on - # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed - # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " - "with parsed arguments; no failed_generation recovery"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "openrouter": LlmProvider( - name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", - url="https://openrouter.ai/api/v1/chat/completions", - model="openai/gpt-4o-mini", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed - # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and - # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), -} - -LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") - - -def mark_no_credit(name, seconds=None): - """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). - - ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for - that to live: a stateless ladder re-tries the empty account on every single turn, pays its - round trip, and shows the reader a longer error each time. This is that memo. - """ - _NO_CREDIT[str(name)] = time.time() + float( - CREDIT_COOLDOWN_S if seconds is None else seconds) - return _NO_CREDIT[str(name)] - - -def no_credit(name): - """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" - until = _NO_CREDIT.get(str(name)) - if not until: - return False - if time.time() >= until: - _NO_CREDIT.pop(str(name), None) - return False - return True - - -def clear_credit_memo(name=None): - """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" - if name is None: - _NO_CREDIT.clear() - else: - _NO_CREDIT.pop(str(name), None) - - -def llm_chain(capability="llm_tool_calling"): - """The provider order for `capability` — declaration first, credit memo second. - - Three filters, in this order, and each removes a DIFFERENT kind of row: - 1. `can()` — declared capable AND configured. An incapable row is never offered, so a - turn cannot be spent discovering it (the `chain()` rule, one layer up). - 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over - until the memo expires. - 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment - can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` - carries for the scraping side. - """ - raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() - names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) - return [LLM_PROVIDERS[n] for n in names - if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] - - -#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND -#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner -#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. -_STATUS_WORDS = { - 401: "{label} would not accept our key", - 403: "{label} would not accept our key", - 402: "{label} is out of credit", - 404: "{label} does not offer the model we asked it for", - 408: "{label} took too long", - 413: "the question was too long for {label}", - 429: "{label} is rate limiting us right now", -} - -#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent -#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the -#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and -#: it only ever decides whether to SKIP a provider, never whether to trust one. -_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", - "quota exceeded", "billing", "payment required", "add credits") - - -def is_credit_failure(status, body=""): - """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" - if int(status or 0) == 402: - return True - if int(status or 0) not in (400, 403, 429): - return False - return any(word in str(body or "").lower() for word in _CREDIT_WORDS) - - -def refusal_sentence(name, status, body=""): - """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. - - ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key - prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a - credit problem", and what reaches the caller is this module's own wording. - """ - label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label - if is_credit_failure(status, body): - return f"{label} is out of credit" - code = int(status or 0) - if code in _STATUS_WORDS: - return _STATUS_WORDS[code].format(label=label) - if 500 <= code <= 599: - return f"{label} is having trouble at their end" - return f"{label} did not answer" - - -# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ -# -# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. -# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation -# drafter) both need it, and the owner quoted an error from EACH of them in one breath: -# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool -# calls are not wired for this shape"*. Two implementations of one wire is -# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the -# ladder that declares the rung. -# -# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: -# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message -# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` -# 3. `temperature` and friends are REMOVED on the current model family -# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string -# -# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on -# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits -# it, the assistant runs on the Opus tier and sends it. - -#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. -ANTHROPIC_VERSION = "2023-06-01" - - -def anthropic_request(*, model, key, system, messages, tools, max_tokens, - tool_choice="auto", effort=None): - """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. - - `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted - out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, - re-addressed rather than re-derived, so a schema change happens in one place. - """ - system_text = "\n\n".join(str(m.get("content") or "") for m in messages - if m.get("role") == "system") - if system: - system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) - turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), - "content": str(m.get("content") or "")} - for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] - wire_tools = [{"name": t["function"]["name"], - "description": t["function"]["description"], - "input_schema": t["function"]["parameters"]} for t in (tools or [])] - body = { - "model": str(model), - "max_tokens": int(max_tokens), - "system": system_text, - "messages": turns, - "tools": wire_tools, - } - # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that - # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, - # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no - # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: - # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. - # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and - # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. - if wire_tools: - body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} - if effort: - body["output_config"] = {"effort": str(effort)} - return {"url": LLM_PROVIDERS["anthropic"].url, - "headers": {"x-api-key": str(key), - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json"}, - "json": body} - - -#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. -#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every -#: call, and a failed import is not cheap. `False` says "asked and answered". -_SDK: object = None - - -def anthropic_sdk(): - """The official `anthropic` SDK, or `None` if this deployment does not carry it. - - ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. - `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's - fence this wave, so the pin is another session's edit. A hard `import anthropic` at module - scope would therefore turn a missing line in a manifest into a container that cannot boot at - all: the API imports this module on every request path. - - So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls - back to the raw `requests` POST that has always worked, and says which transport ran. When the - pin lands the SDK path takes over with no further change. - ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body - `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, - `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer - ([[one-question-two-normalizers]]). - """ - global _SDK - if _SDK is None: - try: - import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring - _SDK = _mod - except Exception: - _SDK = False - return _SDK or None - - -def anthropic_send(req, timeout=None, use_sdk=None): - """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. - - ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader - who cannot tell which path ran cannot tell whether the requirements pin reached the container - [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on - `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a - check keyed to it passes on a stale value written by an earlier check in the same process. - - `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — - the SDK raises where `requests` returns, and normalising that difference here is the only - reason this function exists rather than the caller branching on transport. - - ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports - without assigning to `_SDK`: a test that mutates a module global and then throws leaves every - later check in that process running on the wrong path and passing for the wrong reason. - """ - sdk = None if use_sdk is False else anthropic_sdk() - payload = dict(req.get("json") or {}) - if sdk is None: - r = requests.post(req["url"], headers=req["headers"], json=payload, - timeout=(timeout or 60)) - try: - return r.status_code, (r.json() if r.content else {}), "http" - except Exception: - return r.status_code, {"_text": (r.text or "")[:2000]}, "http" - - key = (req.get("headers") or {}).get("x-api-key") or "" - try: - client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) - msg = client.messages.create(**payload) - # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same - # wire-shaped dict the raw POST parses out of the response body. - return 200, msg.model_dump(), "sdk" - except Exception as exc: - # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by - # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out - # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. - status = int(getattr(exc, "status_code", 0) or 0) - body = getattr(exc, "body", None) - if not isinstance(body, dict): - body = {"_text": str(exc)[:2000]} - if not status: - # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one - # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. - status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 - return status, body, "sdk" - - -def anthropic_read(body): - """`(text, tool_input, refusal)` out of a Messages API answer. - - ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with - `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` - unconditionally breaks on exactly the turn a person most needs explained. - ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and - no regex recovering a call out of a 400, which is what the other wire needs. - """ - body = body if isinstance(body, dict) else {} - if str(body.get("stop_reason") or "") == "refusal": - return "", None, "the assistant declined to answer that one" - blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] - text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() - calls = [b for b in blocks if b.get("type") == "tool_use"] - got = calls[0].get("input") if calls else None - return text, (got if isinstance(got, dict) else None), None - - -def llm_status(capability="llm_tool_calling"): - """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. - - ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read - THIS, so a model offered in one place cannot be missing from the other, and neither can offer a - provider the ladder would refuse to call [[permitted-is-not-answerable]]. - """ - rows = [] - for name in LLM_DEFAULT_ORDER: - p = LLM_PROVIDERS[name] - cap = p.caps.get(capability) - rows.append({ - "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, - "configured": p.configured(), - "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), - "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), - "capable": bool(cap and cap.capable), - "outOfCredit": no_credit(name), - "note": (cap.note if cap else ""), - }) - return rows - - -# ============================================================================================= -# THE CANONICAL SCHEMA — owner ruling 2026-08-08: -# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set -# database even if the underlying engine changes"* -# -# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and -# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a -# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves -# with the vendor turns a routing change into a data migration. -# -# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. -# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so -# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column -# nobody declared. -# -# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded -# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a -# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is -# exactly the distinction that let an account-grain number wear the "Views" label for a month. -# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH -# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count -# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk -# fields AGREE with each other, which is precisely what makes picking by name so dangerous. -# canonical `views` <- apify `videoPlayCount` ✅ matches the grid -# canonical `views` <- apify `videoViewCount` ⛔ off by 13x -# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain -# ============================================================================================= - -#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider -#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier -#: paid run learned. -CANONICAL_POST_KEYS = ( - "shortcode", "url", "influencer_key", "posted_at", "type", "caption", - "likes", "comments", "views", "paid_partnership", "partner", "hashtags", - "alt_text", "tagged_location", "source_payload", - # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a - # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a - # value that normalises cleanly and is then dropped by the write door, silently. - "plays", "video_duration", "comments_disabled", -) - - -def _int_or_none(v): - """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like - count — a real state that is not a measurement. Writing -1 would render as a negative like - count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" - try: - n = int(v) - except (TypeError, ValueError): - return None - return None if n < 0 else n - - -def normalize_post_apify(row): - """One Apify `instagram-scraper` item -> the canonical post row.""" - if not isinstance(row, dict): - return None - code = str(row.get("shortCode") or "").strip() - if not code: - return None - out = { - "shortcode": code, - "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), - "influencer_key": str(row.get("ownerUsername") or ""), - "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], - "type": "video" if str(row.get("type") or "").lower() == "video" else - ("carousel" if row.get("childPosts") else "image"), - "caption": str(row.get("caption") or ""), - # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: - # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true - # count was ~137K, and `videoPlayCount` matched a browser read to the digit. - "views": _int_or_none(row.get("videoPlayCount")), - # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). - # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing - # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: - # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different - # real numbers, and we were storing only one of them. - # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two - # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: - # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured - # mean, and a column whose meaning changes halfway down is the one thing worse than a - # column with no data. Flagged rather than done. - "plays": _int_or_none(row.get("videoPlayCount")), - "likes": _int_or_none(row.get("likesCount")), - "comments": _int_or_none(row.get("commentsCount")), - # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in - # this same response (owner: *"whatever APIfy has more than BD pls use it"*). - "video_duration": _int_or_none(row.get("videoDuration")), - "comments_disabled": "1" if row.get("isCommentsDisabled") else "", - "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), - "alt_text": str(row.get("alt") or ""), - "paid_partnership": "1" if row.get("paidPartnership") else "", - "partner": ", ".join(str((s or {}).get("username") or "") - for s in (row.get("sponsors") or []) if isinstance(s, dict)), - "tagged_location": str((row.get("locationName") or "")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - return {k: v for k, v in out.items() if v not in (None, "")} - - -def normalize_profile_apify(row): - """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. - - ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly - in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and - `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran - that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify - was never asked, so a profile Bright Data cannot scrape came back `blocked` while a - configured, declared-capable provider sat unused. A registry entry with no runner is a - promise the product does not keep ([[flag-shipped-without-its-writer]]). - - ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module - already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the - snapshot writer cannot tell which vendor answered — which is what makes the fallback - invisible to every consumer instead of a second schema. - - ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` - drops a missing count: writing 0 followers would claim we measured an empty account, and the - caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would - stop the chain on a lie. - """ - if not isinstance(row, dict): - return None - # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle - # with a 200 and a well-formed item — `{"username": …, "error": "not_found", - # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" - # out of it: a username, a url and a source_payload, with every measured field absent. It - # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. - # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this - # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's - # ERROR shape is the one that must never become a row. - # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on - # `sriyynntt`), so `"error" in row` would refuse every good profile. - if row.get("error"): - return None - handle = str(row.get("username") or "").strip() - if not handle: - return None - out = { - "username": handle, - "full_name": str(row.get("fullName") or ""), - "bio": str(row.get("biography") or ""), - "followers": _int_or_none(row.get("followersCount")), - "following": _int_or_none(row.get("followsCount")), - "posts_count": _int_or_none(row.get("postsCount")), - "verified": "1" if row.get("verified") else "", - "external_url": str(row.get("externalUrl") or ""), - "ig_id": str(row.get("id") or ""), - "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), - "business_category": str(row.get("businessCategoryName") or ""), - "is_business": "1" if row.get("isBusinessAccount") else "", - "is_private": "1" if row.get("private") else "", - "highlights_count": _int_or_none(row.get("highlightReelCount")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter - # below must not treat 0 as absent the way the post normaliser can — an account really can - # have zero followers. Only None and "" are dropped. - return {k: v for k, v in out.items() if v is not None and v != ""} - - -def canonical_gaps(row, want=("views",)): - """Which requested canonical fields this row does NOT carry — the `satisfied` input. - - Named rather than inlined because "did the vendor actually answer the question" is the whole - fallback trigger, and a chain that asks it differently in two places will drift. - """ - row = row if isinstance(row, dict) else {} - return [k for k in want if row.get(k) in (None, "")] +"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. + +⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if +anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize +when we scale to thousands+ customer … make sure our code is customizable and modular."* + +⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". +A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious +shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments +correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A +provider-level fallback would notice the missing views and re-buy the profile, the likes and the +comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that +is noise; at thousands of customers × twelve posts each it is the whole bill. + +So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: + + ig_profile -> brightdata (36 fields, works) + ig_post_metrics -> brightdata (likes + comments, works) + ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) + ig_comments -> brightdata + +Each capability has an ORDERED list of providers. The first one that is configured AND capable AND +succeeds wins; the rest are never called, so the common path costs exactly one vendor record. + +⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in +the one column you asked for has failed for our purposes, and a chain that only catches exceptions +would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying +answer exactly as it would on a 500. + +⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time +(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what +lets a tenant be moved off a vendor without a release. +""" +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Callable + +# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in +# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the +# line that stops a container booting. That asymmetry is the whole reason the two are imported +# differently — see `anthropic_sdk()`. +import requests + +#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK +#: providers and to estimate a run's spend for the operator. They are not billing truth — the +#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is +#: why they are named and dated rather than guessed silently. +#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper +#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. +#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number +#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before +#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. +#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here +#: reads a live price — a modelled figure that says so is honest; one that pretends is not. +_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) +_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) + + +@dataclass(frozen=True) +class Capability: + """What ONE provider can do for ONE capability, and what it costs to ask.""" + #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never + #: called for this capability at all — it cannot be reached by a fallback, cannot be put first + #: by an env override, and cannot silently start being billed because somebody reordered a + #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route + #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four + #: accounts from 13K to 268M followers returned an account-grain constant or a null. + capable: bool = True + cost_per_record: float = 0.0 + #: Free-text, shown to an operator deciding where their money went. + note: str = "" + + +@dataclass +class Provider: + key: str + label: str + #: The env var holding this provider's credential. NEVER the credential itself — this module + #: is imported by surfaces that serialise their config. + key_env: str = "" + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True + + def cap(self, capability: str) -> Capability | None: + return self.caps.get(capability) + + def can(self, capability: str) -> bool: + c = self.cap(capability) + return bool(c and c.capable and self.configured()) + + +#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the +#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. +PROVIDERS: dict[str, Provider] = { + "brightdata": Provider( + key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", + caps={ + "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), + "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), + "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), + # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. + "ig_post_views": Capability( + False, _COST_BRIGHTDATA, + "MEASURED INCAPABLE: returns one account-grain number for every reel of a " + "creator (identical for two shortcodes in one call, ticking upward between " + "calls) and never populates video_play_count. Their own documented example " + "account no longer reproduces it."), + # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our + # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own + # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. + "tt_profile": Capability(True, _COST_BRIGHTDATA, + "40 fields incl. bio/engagement rates/region"), + # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. + # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says + # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post + # record and a separate view rung would be a second bill for a number we already have. + # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: + # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an + # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it + # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a + # quiet fallback bolted onto this one. + "tt_post_metrics": Capability( + True, _COST_BRIGHTDATA, + "likes + comments + shares + saves, and play_count inline (DECLARED " + "no-empties-or-zeros, UNPROVEN until one live pull)"), + "tt_comments": Capability(True, _COST_BRIGHTDATA, + "separate paid dataset, opt-in: 17 fields"), + }), + "apify": Provider( + key="apify", label="Apify", key_env="AIOS_APIFY_KEY", + caps={ + # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and + # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. + "ig_post_views": Capability(True, _COST_APIFY, + "videoPlayCount, matches Instagram's displayed views"), + "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), + "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), + }), +} + +#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. +DEFAULT_CHAINS: dict[str, tuple] = { + "ig_profile": ("brightdata", "apify"), + "ig_post_metrics": ("brightdata", "apify"), + # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here + # would be a lie that costs a wasted call on every single post. + "ig_post_views": ("apify",), + "ig_comments": ("brightdata",), + # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. + # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a + # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data + # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap + # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here + # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, + # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; + # it is not declared until a runner walks it. + "tt_profile": ("brightdata",), + "tt_post_metrics": ("brightdata",), + "tt_comments": ("brightdata",), +} + + +def chain(capability: str) -> list: + """The provider order for `capability` — env override first, then the default. + + `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the + owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a + vendor that is having a bad day. + ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — + `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore + resolves to EMPTY rather than to a provider that would return a wrong number, because a + plausible wrong number is worse than an honest refusal. + """ + raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() + names = None + for clause in raw.split(";"): + if "=" in clause: + cap_name, _, order = clause.partition("=") + if cap_name.strip() == capability: + names = [x.strip() for x in order.split(",") if x.strip()] + if names is None: + names = list(DEFAULT_CHAINS.get(capability) or ()) + return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] + + +def estimate(capability: str, records: int) -> dict: + """What the FIRST capable provider would cost for `records` — the number an operator plans on. + + Reported per capability rather than per run because that is the unit that scales: at a thousand + tenants the question is never "what did this run cost", it is "what does adding view counts to + every post cost per month". + """ + ch = chain(capability) + if not ch: + return {"capability": capability, "records": records, "provider": None, "usd": 0.0, + "note": "no configured, capable provider"} + p = ch[0] + c = p.cap(capability) + return {"capability": capability, "records": records, "provider": p.key, + "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), + "note": c.note} + + +@dataclass +class Attempt: + provider: str + ok: bool + note: str = "" + records: int = 0 + seconds: float = 0.0 + + +def run(capability, work, satisfied=None, log=None): + """Walk the chain for `capability` until one provider gives a SATISFYING answer. + + `work(provider) -> (result, note)`; a non-empty note means it did not answer. + `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any + truthy result. + + Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it + satisfied, and how long it took — because "where did this number come from and what did it + cost" is a question somebody asks about a bill, not about a stack trace. + + ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the + failure it must catch is not an error: a vendor answering 200 with the one field you needed + left blank. That is exactly how Bright Data behaves on view counts, and a chain that only + caught exceptions would have stopped there forever and never reached Apify. + """ + log = log or (lambda *_a: None) + ok = satisfied or (lambda r: bool(r)) + attempts, last = [], None + for provider in chain(capability): + started = time.time() + try: + result, note = work(provider) + except Exception as e: # noqa: BLE001 + # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is + # one refactor away from being a query param; this line must not be what leaks it. + result, note = None, f"{type(e).__name__}" + secs = round(time.time() - started, 2) + if note: + attempts.append(Attempt(provider.key, False, note, 0, secs)) + log(f" {provider.label}: {note}, falling through") + continue + if not ok(result): + attempts.append(Attempt(provider.key, False, "answered without the field asked for", + 0, secs)) + log(f" {provider.label}: answered, but not with what was asked for, falling through") + last = result if last is None else last + continue + attempts.append(Attempt(provider.key, True, "", _count(result), secs)) + log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") + return result, attempts + return last, attempts + + +def _count(result): + if isinstance(result, (list, tuple, set)): + return len(result) + if isinstance(result, dict): + return len(result) + return 1 if result else 0 + + +def wire(): + """What the Settings surface renders. Booleans and labels — NEVER a credential. + + Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a + provider is usable, never what the key is. + """ + return { + "providers": [ + {"key": p.key, "label": p.label, "configured": p.configured(), + "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} + for p in PROVIDERS.values() + ], + "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, + "incapable": { + f"{p.key}:{cap}": c.note + for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable + }, + } + + + + +# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ +# +# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the +# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: +# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this +# shape'."* +# +# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the +# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is +# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a +# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. +# +# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is +# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is +# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half +# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the +# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both +# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. +# +# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that +# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. +# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling +# turn at all, so the guess never has to be made. + +#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: +#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen +#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short +#: enough that a top-up is picked up without a restart. +CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) + +#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a +#: latency optimisation, not a billing record. A second container learns the same thing from its +#: own first 402, and neither one can be wrong for longer than the cooldown. +_NO_CREDIT: dict[str, float] = {} + + +@dataclass(frozen=True) +class LlmProvider: + """One chat-completions endpoint, and what it is DECLARED able to do.""" + name: str + label: str + env: str + url: str + model: str + #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, + #: which is a different body, a different auth header and a different result shape. + wire: str + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.env) or "").strip()) + + def can(self, capability: str) -> bool: + cap = self.caps.get(capability) + return bool(cap and cap.capable and self.configured()) + + +#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put +#: cerebras first *"because this path needs tool calling and cerebras carries this account's +#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always +#: works, and the others are the cheap seats it falls through to. +LLM_PROVIDERS: dict[str, LlmProvider] = { + "anthropic": LlmProvider( + name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", + url="https://api.anthropic.com/v1/messages", + # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with + # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier + # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real + # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. + model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", + wire="anthropic", + caps={ + # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a + # typed `tool_use` content block carrying parsed `input`. There is nothing to recover + # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to + # apologise for on the other wire. + "llm_tool_calling": Capability(True, 0.0, + "typed tool_use content block; no text recovery path"), + "llm_chat": Capability(True, 0.0, "Messages API"), + "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), + }), + "cerebras": LlmProvider( + name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", + url="https://api.cerebras.ai/v1/chat/completions", + # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the + # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on + # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` + # already knows how to survive (R4: skipped, not tried). + model="gpt-oss-120b", wire="openai", + caps={ + # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not + # measure this rung: a 402 comes back before any tool is considered, so there is no + # observation to record. Leaving the old wording is the honest answer — upgrading it + # to MEASURED beside its two neighbours would be claiming an account balance as + # evidence about a capability. + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured: this account's tool-capable " + "model per the wave-32 ladder note. ⚠ 2026-08-19: the " + "account answers 402, so this stays unmeasured"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "groq": LlmProvider( + name="groq", label="Groq", env="GROQ_API_KEY", + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq + # RETIRED it. Measured, not inferred — a real POST with the live key answered + # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with + # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A + # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the + # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). + # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. + # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while + # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY + # spelled differently and a sweep that "normalises" them breaks one of them. + url="https://api.groq.com/openai/v1/chat/completions", + model="openai/gpt-oss-120b", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old + # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME + # provider on this wire answers 400 with the call in `failed_generation`; the repo + # never recorded which. Flip this to False the day it is"*. It is recorded now: on + # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed + # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " + "with parsed arguments; no failed_generation recovery"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "openrouter": LlmProvider( + name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o-mini", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed + # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and + # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), +} + +LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") + + +def mark_no_credit(name, seconds=None): + """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). + + ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for + that to live: a stateless ladder re-tries the empty account on every single turn, pays its + round trip, and shows the reader a longer error each time. This is that memo. + """ + _NO_CREDIT[str(name)] = time.time() + float( + CREDIT_COOLDOWN_S if seconds is None else seconds) + return _NO_CREDIT[str(name)] + + +def no_credit(name): + """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" + until = _NO_CREDIT.get(str(name)) + if not until: + return False + if time.time() >= until: + _NO_CREDIT.pop(str(name), None) + return False + return True + + +def clear_credit_memo(name=None): + """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" + if name is None: + _NO_CREDIT.clear() + else: + _NO_CREDIT.pop(str(name), None) + + +def llm_chain(capability="llm_tool_calling"): + """The provider order for `capability` — declaration first, credit memo second. + + Three filters, in this order, and each removes a DIFFERENT kind of row: + 1. `can()` — declared capable AND configured. An incapable row is never offered, so a + turn cannot be spent discovering it (the `chain()` rule, one layer up). + 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over + until the memo expires. + 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment + can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` + carries for the scraping side. + """ + raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() + names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) + return [LLM_PROVIDERS[n] for n in names + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + + +#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND +#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner +#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. +_STATUS_WORDS = { + 401: "{label} would not accept our key", + 403: "{label} would not accept our key", + 402: "{label} is out of credit", + 404: "{label} does not offer the model we asked it for", + 408: "{label} took too long", + 413: "the question was too long for {label}", + 429: "{label} is rate limiting us right now", +} + +#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent +#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the +#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and +#: it only ever decides whether to SKIP a provider, never whether to trust one. +_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", + "quota exceeded", "billing", "payment required", "add credits") + + +def is_credit_failure(status, body=""): + """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" + if int(status or 0) == 402: + return True + if int(status or 0) not in (400, 403, 429): + return False + return any(word in str(body or "").lower() for word in _CREDIT_WORDS) + + +def refusal_sentence(name, status, body=""): + """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. + + ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key + prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a + credit problem", and what reaches the caller is this module's own wording. + """ + label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label + if is_credit_failure(status, body): + return f"{label} is out of credit" + code = int(status or 0) + if code in _STATUS_WORDS: + return _STATUS_WORDS[code].format(label=label) + if 500 <= code <= 599: + return f"{label} is having trouble at their end" + return f"{label} did not answer" + + +# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ +# +# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. +# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation +# drafter) both need it, and the owner quoted an error from EACH of them in one breath: +# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool +# calls are not wired for this shape"*. Two implementations of one wire is +# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the +# ladder that declares the rung. +# +# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: +# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message +# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` +# 3. `temperature` and friends are REMOVED on the current model family +# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string +# +# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on +# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits +# it, the assistant runs on the Opus tier and sends it. + +#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. +ANTHROPIC_VERSION = "2023-06-01" + + +def anthropic_request(*, model, key, system, messages, tools, max_tokens, + tool_choice="auto", effort=None): + """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. + + `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted + out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, + re-addressed rather than re-derived, so a schema change happens in one place. + """ + system_text = "\n\n".join(str(m.get("content") or "") for m in messages + if m.get("role") == "system") + if system: + system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) + turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), + "content": str(m.get("content") or "")} + for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + wire_tools = [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])] + body = { + "model": str(model), + "max_tokens": int(max_tokens), + "system": system_text, + "messages": turns, + "tools": wire_tools, + } + # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that + # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, + # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no + # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: + # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. + # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and + # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. + if wire_tools: + body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} + if effort: + body["output_config"] = {"effort": str(effort)} + return {"url": LLM_PROVIDERS["anthropic"].url, + "headers": {"x-api-key": str(key), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + "json": body} + + +#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. +#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every +#: call, and a failed import is not cheap. `False` says "asked and answered". +_SDK: object = None + + +def anthropic_sdk(): + """The official `anthropic` SDK, or `None` if this deployment does not carry it. + + ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. + `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's + fence this wave, so the pin is another session's edit. A hard `import anthropic` at module + scope would therefore turn a missing line in a manifest into a container that cannot boot at + all: the API imports this module on every request path. + + So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls + back to the raw `requests` POST that has always worked, and says which transport ran. When the + pin lands the SDK path takes over with no further change. + ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body + `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, + `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer + ([[one-question-two-normalizers]]). + """ + global _SDK + if _SDK is None: + try: + import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring + _SDK = _mod + except Exception: + _SDK = False + return _SDK or None + + +def anthropic_send(req, timeout=None, use_sdk=None): + """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. + + ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader + who cannot tell which path ran cannot tell whether the requirements pin reached the container + [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + check keyed to it passes on a stale value written by an earlier check in the same process. + + `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — + the SDK raises where `requests` returns, and normalising that difference here is the only + reason this function exists rather than the caller branching on transport. + + ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports + without assigning to `_SDK`: a test that mutates a module global and then throws leaves every + later check in that process running on the wrong path and passing for the wrong reason. + """ + sdk = None if use_sdk is False else anthropic_sdk() + payload = dict(req.get("json") or {}) + if sdk is None: + r = requests.post(req["url"], headers=req["headers"], json=payload, + timeout=(timeout or 60)) + try: + return r.status_code, (r.json() if r.content else {}), "http" + except Exception: + return r.status_code, {"_text": (r.text or "")[:2000]}, "http" + + key = (req.get("headers") or {}).get("x-api-key") or "" + try: + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + msg = client.messages.create(**payload) + # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same + # wire-shaped dict the raw POST parses out of the response body. + return 200, msg.model_dump(), "sdk" + except Exception as exc: + # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by + # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out + # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. + status = int(getattr(exc, "status_code", 0) or 0) + body = getattr(exc, "body", None) + if not isinstance(body, dict): + body = {"_text": str(exc)[:2000]} + if not status: + # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one + # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. + status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 + return status, body, "sdk" + + +def anthropic_read(body): + """`(text, tool_input, refusal)` out of a Messages API answer. + + ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with + `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` + unconditionally breaks on exactly the turn a person most needs explained. + ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and + no regex recovering a call out of a 400, which is what the other wire needs. + """ + body = body if isinstance(body, dict) else {} + if str(body.get("stop_reason") or "") == "refusal": + return "", None, "the assistant declined to answer that one" + blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] + text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() + calls = [b for b in blocks if b.get("type") == "tool_use"] + got = calls[0].get("input") if calls else None + return text, (got if isinstance(got, dict) else None), None + + +def llm_status(capability="llm_tool_calling"): + """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. + + ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read + THIS, so a model offered in one place cannot be missing from the other, and neither can offer a + provider the ladder would refuse to call [[permitted-is-not-answerable]]. + """ + rows = [] + for name in LLM_DEFAULT_ORDER: + p = LLM_PROVIDERS[name] + cap = p.caps.get(capability) + rows.append({ + "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, + "configured": p.configured(), + "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), + "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), + "capable": bool(cap and cap.capable), + "outOfCredit": no_credit(name), + "note": (cap.note if cap else ""), + }) + return rows + + +# ============================================================================================= +# THE CANONICAL SCHEMA — owner ruling 2026-08-08: +# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set +# database even if the underlying engine changes"* +# +# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and +# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a +# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves +# with the vendor turns a routing change into a data migration. +# +# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. +# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so +# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column +# nobody declared. +# +# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded +# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a +# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is +# exactly the distinction that let an account-grain number wear the "Views" label for a month. +# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH +# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count +# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk +# fields AGREE with each other, which is precisely what makes picking by name so dangerous. +# canonical `views` <- apify `videoPlayCount` ✅ matches the grid +# canonical `views` <- apify `videoViewCount` ⛔ off by 13x +# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain +# ============================================================================================= + +#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider +#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier +#: paid run learned. +CANONICAL_POST_KEYS = ( + "shortcode", "url", "influencer_key", "posted_at", "type", "caption", + "likes", "comments", "views", "paid_partnership", "partner", "hashtags", + "alt_text", "tagged_location", "source_payload", + # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a + # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a + # value that normalises cleanly and is then dropped by the write door, silently. + "plays", "video_duration", "comments_disabled", +) + + +def _int_or_none(v): + """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like + count — a real state that is not a measurement. Writing -1 would render as a negative like + count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" + try: + n = int(v) + except (TypeError, ValueError): + return None + return None if n < 0 else n + + +def normalize_post_apify(row): + """One Apify `instagram-scraper` item -> the canonical post row.""" + if not isinstance(row, dict): + return None + code = str(row.get("shortCode") or "").strip() + if not code: + return None + out = { + "shortcode": code, + "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), + "influencer_key": str(row.get("ownerUsername") or ""), + "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], + "type": "video" if str(row.get("type") or "").lower() == "video" else + ("carousel" if row.get("childPosts") else "image"), + "caption": str(row.get("caption") or ""), + # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: + # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true + # count was ~137K, and `videoPlayCount` matched a browser read to the digit. + "views": _int_or_none(row.get("videoPlayCount")), + # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). + # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing + # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: + # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different + # real numbers, and we were storing only one of them. + # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two + # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: + # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured + # mean, and a column whose meaning changes halfway down is the one thing worse than a + # column with no data. Flagged rather than done. + "plays": _int_or_none(row.get("videoPlayCount")), + "likes": _int_or_none(row.get("likesCount")), + "comments": _int_or_none(row.get("commentsCount")), + # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in + # this same response (owner: *"whatever APIfy has more than BD pls use it"*). + "video_duration": _int_or_none(row.get("videoDuration")), + "comments_disabled": "1" if row.get("isCommentsDisabled") else "", + "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), + "alt_text": str(row.get("alt") or ""), + "paid_partnership": "1" if row.get("paidPartnership") else "", + "partner": ", ".join(str((s or {}).get("username") or "") + for s in (row.get("sponsors") or []) if isinstance(s, dict)), + "tagged_location": str((row.get("locationName") or "")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + return {k: v for k, v in out.items() if v not in (None, "")} + + +def normalize_profile_apify(row): + """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. + + ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly + in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and + `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran + that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify + was never asked, so a profile Bright Data cannot scrape came back `blocked` while a + configured, declared-capable provider sat unused. A registry entry with no runner is a + promise the product does not keep ([[flag-shipped-without-its-writer]]). + + ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module + already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the + snapshot writer cannot tell which vendor answered — which is what makes the fallback + invisible to every consumer instead of a second schema. + + ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` + drops a missing count: writing 0 followers would claim we measured an empty account, and the + caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would + stop the chain on a lie. + """ + if not isinstance(row, dict): + return None + # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle + # with a 200 and a well-formed item — `{"username": …, "error": "not_found", + # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" + # out of it: a username, a url and a source_payload, with every measured field absent. It + # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. + # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this + # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's + # ERROR shape is the one that must never become a row. + # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on + # `sriyynntt`), so `"error" in row` would refuse every good profile. + if row.get("error"): + return None + handle = str(row.get("username") or "").strip() + if not handle: + return None + out = { + "username": handle, + "full_name": str(row.get("fullName") or ""), + "bio": str(row.get("biography") or ""), + "followers": _int_or_none(row.get("followersCount")), + "following": _int_or_none(row.get("followsCount")), + "posts_count": _int_or_none(row.get("postsCount")), + "verified": "1" if row.get("verified") else "", + "external_url": str(row.get("externalUrl") or ""), + "ig_id": str(row.get("id") or ""), + "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), + "business_category": str(row.get("businessCategoryName") or ""), + "is_business": "1" if row.get("isBusinessAccount") else "", + "is_private": "1" if row.get("private") else "", + "highlights_count": _int_or_none(row.get("highlightReelCount")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter + # below must not treat 0 as absent the way the post normaliser can — an account really can + # have zero followers. Only None and "" are dropped. + return {k: v for k, v in out.items() if v is not None and v != ""} + + +def canonical_gaps(row, want=("views",)): + """Which requested canonical fields this row does NOT carry — the `satisfied` input. + + Named rather than inlined because "did the vendor actually answer the question" is the whole + fallback trigger, and a chain that asks it differently in two places will drift. + """ + row = row if isinstance(row, dict) else {} + return [k for k in want if row.get(k) in (None, "")] diff --git a/api/routes_admin.py b/api/routes_admin.py index fd2e72ac98ee187a4163f92606d564f0a96a00a7..2c4d566ff5b952262589daff493441255494beba 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -546,6 +546,203 @@ def _perm_modules(session, surfaces=False): #: editor renders, so "shown" and "storable" cannot drift apart again. +#: What a picker row says about provenance when nothing could be established: no author on the +#: definition, no owner in the registry, and no answer to "does that owner hold admin". +#: ⛔⛔ `ownerIsAdmin: None` IS NOT `False`, AND THE DIFFERENCE IS THE WHOLE POINT OF W41-T08. +#: `False` asserts "a known account that is not an administrator owns this column"; `None` says +#: "unresolved". R8 arms an ADMIN-CREATED-ONLY rule on top of this, and a consumer that read the +#: unresolved answer as `False` would wall a column on a store outage — silently, and in the +#: direction that breaks a live permission rule. Every consumer must treat `None` as "do not act". +#: ⚠ `origin` FALLS TO `"preset"` for a row with no definition behind it (the `measure_` +#: pseudo-fields), which is the truth rather than a default: the platform's metric catalogue made +#: those, not a person. `field_origin`'s own docstring argues the same polarity. +_PROVENANCE_UNKNOWN = {"createdBy": None, "owner": None, "ownerIsAdmin": None, + "origin": "preset"} + +#: ⭐⭐ W41-T09 / RULING R8 / OWNER INSTRUCTION 15 — THE KEY A PICKER ROW CARRIES WHEN R8 IS WHAT +#: TOOK IT OUT OF THE PERMISSION FILTER BUILDER. A MODULE CONSTANT rather than a literal, because +#: the producer (`_module_fields`) and the consumer (`_clean_perms`' named refusal) sit 500 lines +#: apart: two spellings of one key is wave 40's own scar, where a TITLE instruction shipped as a +#: no-op with both lanes correct and every gate green because the consumer never read the +#: producer's new name [[two-lanes-one-contract-dead-feature]]. +#: ⚠ PRESENT ONLY WHEN R8 IS THE CAUSE. A `measure_` pseudo-field and `est_missed` are +#: `filterable: False` for reasons that predate this rule and carry no reason key, so the set of +#: rows holding one IS exactly the set R8 refuses — which is what lets the validator derive its +#: refusal from the very rows the picker was built from instead of recomputing the predicate. +_FILTER_REASON_KEY = "filterableReason" + +#: ⛔ USER-FACING COPY (CONTRACT C7): no em dash, no en dash, no ALL-CAPS chrome. One sentence, and +#: it NAMES THE OWNER because that is the repair rather than a diagnosis with no next step — +#: W41-T03 shipped owner reassignment this wave, so "who owns it" is the actionable half. +_ADMIN_ONLY_FILTER_REASON = ("Only a column owned by an administrator can be used in a permission " + "filter, and {owner} owns this one.") + + +def _apply_admin_only_filter_rule(row): + """RULING R8, ARMED: a column whose CURRENT OWNER does not hold admin leaves the permission + filter builder, and says why. + + Owner instruction 15: *"only admin-created fields may be used for permissioning filtration."* + ⚠ THE PREMISE HAD NO SUBJECT UNTIL W41-T08 — a picker row carried `custom` and nothing else — + so R8 defines "admin-created" as THE FIELD'S CURRENT OWNER HOLDS THE ADMIN ROLE, repairable + through the ownership transfer W41-T03 shipped. On live today the two readings coincide: + `object_shares` holds no `field` namespace at all for this tenant, so every owner resolves + through C1's documented `createdBy` fallback (W41-T08's census, measured 2026-08-24). + + ⛔⛔ `None` ADMITS, AND THAT DECISION IS THE WHOLE SAFETY PROPERTY OF THIS FUNCTION. + `ownerIsAdmin` is `False` only for a KNOWN account known not to hold admin; it is `None` for + "unresolved" — no owner recorded, the roster unreadable, or no session to read either with. + `store.get` answers `{}` on a transient failure, so a rule that refused on `None` would revoke + EVERY live permission filter in the tenant at once, off one bad read, in the direction that + hands an account the whole book. Admitting on `None` costs the opposite and it is bounded: a + degraded read leaves the offer exactly as wide as it was the day before R8 shipped. + ⚠ AND THE CHOICE IS MEASURED, NOT ONLY ARGUED. `verify_api`'s W40-T18 fixture + `custom_region_qa` carries no `createdBy` and no grant record, so it resolves `None`; refusing + on `None` turns that shipped ACCEPTANCE leg red. The safe direction and the measured one agree. + + ⛔ NARROW THREE WAYS, and each one names a live column this must not touch: + + * `origin == "user"` ONLY, and the badge comes from CONTRACT C1 (`field_origin`), never from + a second derivation in this file. A PLATFORM column is not "admin-created" either on a + literal reading, and walling `dba` or `revenue` because somebody was handed their + ownership would be R8 eating the database's own contract. + * A row that is ALREADY unfilterable is left alone, reason key and all. A `measure_` + pseudo-field and `est_missed` are out of the builder for causes that predate R8, and + stamping this sentence on them would explain their absence with the wrong reason. + * The FILTER only. ⛔ R8 ARMS FILTRATION, NOT HIDING: `_clean_perms` validates + `hiddenFields` against EVERY key and the permanent filter against `filterable` alone, so + flipping this one flag reaches the builder and leaves the hide list whole. The nine + declared `shared` product columns and all five of these stay hideable. + + ⚠ A KNOWN GAP, NAMED RATHER THAN PAPERED OVER: on a `ut_*` database `user_tables._clean_field` + is a strict allowlist that keeps neither `custom` nor `createdBy`, so its columns answer + `origin: "preset"` and R8 cannot fire there until a transfer writes a real grant record. + `field_origin`'s own docstring books that allowlist as the defect; this inherits the gap rather + than routing around it with the second badge C1 exists to forbid. + + ⛔ THE STORED RECORD IS NOT TOUCHED, and that is deliberate. `get_perms`' I16 cascade prunes a + leaf naming a column the database no longer HAS; a column that still exists and merely changed + who may name it is not stale, and pruning it would DELETE a live permission rule silently, in + the WIDENING direction. An existing wall keeps working (`apply_row_scope` is unchanged) and a + re-save of it is refused LOUDLY by `_clean_perms`, with the column named. + + ⚠ `owner` IS GUARDED STRUCTURALLY, not defensively: `_field_provenance` cannot answer `False` + without one, and the sentence is only well formed with a name in it. + """ + owner = row.get("owner") + if (row.get("origin") != "user" or not row.get("filterable") + or row.get("ownerIsAdmin") is not False or not owner): + return row + return dict(row, filterable=False, + **{_FILTER_REASON_KEY: _ADMIN_ONLY_FILTER_REASON.format(owner=owner)}) + + +def _admin_usernames(): + """Every account holding the `admin` role, lowercased — or `None` when that set could not be + established. + + ⛔⛔ `None` IS NOT `set()`, for the reason `perm_scope.user_generated_fields` states one door + over: `set()` means "resolved: nobody here is an administrator" and `None` means "the roster + could not be read". `core.users.registry` is a LENIENT display read (`store.get`), so a store + outage hands back `{}` — indistinguishable from a tenant with no accounts, and reading that as + "no admins" would make every `ownerIsAdmin` answer `False` at exactly the moment nothing can be + verified. An EMPTY registry therefore answers `None`; a populated one with no admin in it is a + real (and reportable) `set()`. + + ⚠ THE REGISTRY IS A GLOBAL CONTROL-PLANE BUCKET AND THAT IS WHY THIS IS NOT TENANT-SCOPED. + `list_users` says it in full: one bucket holds every tenant's accounts, keyed by username, so a + username resolves to exactly one record and exactly one role however the caller arrived. The + tenant filter there protects a ROSTER LISTING; this asks a per-username question that has one + answer. + """ + try: + reg = users.registry() or {} + except Exception: # noqa: BLE001 + return None + if not isinstance(reg, dict) or not reg: + return None + return {str(u).strip().lower() for u, rec in reg.items() + if isinstance(rec, dict) + and str(rec.get("role") or "user").strip().lower() == "admin"} + + +def _field_provenance(src, key, session): + """`{field_key: {createdBy, owner, ownerIsAdmin, origin}}` for the definitions in `src`. + + ⭐⭐ W41-T08 / RULING R8 / CONTRACT C1 — WHO MADE THIS COLUMN, WHO OWNS IT NOW, AND WHETHER + THAT OWNER HOLDS ADMIN. R8 wants permission filters restricted to admin-created columns, and + the premise could not even be CHECKED before this: a picker row carried `custom` and nothing + else, so "admin-created" had no subject on the wire. ⛔ THIS FUNCTION MEASURES AND REPORTS. + It arms nothing, filters nothing and drops nothing — W41-T09 is the ticket that decides what a + wall does with the answer, and it is blocked on the census this produces. + + ⛔ THE OWNER COMES FROM CONTRACT C1 AND FROM NOWHERE ELSE. `field_permissions.field_classes` is + the ONE producer of a field's badges (`{origin, audience, sharedBy, values, owner}`) and the + permission-rule picker is one of its four named consumers. Re-reading `object_shares` here, or + treating `createdBy` as the owner, would be the second derivation C1 exists to forbid — and the + two would disagree the first time a column changed hands, which is precisely the question R8 + asks. `createdBy` still travels, but as the RAW author stamp beside the owner, never as it. + + ⚠ THE BATCH, NEVER `field_class` IN A LOOP. `field_classes` opens the grant registry ONCE for + the whole list; the per-field door deep-copies `object_shares` per column, which is the D-214 + shape its own docstring spent a ticket removing. This runs on every admin GET. + + ⚠ `values_shared=()` IS DELIBERATE AND IT IS WHY ONLY TWO MEMBERS OF THE BAG ARE READ HERE. + The `values` and `audience` badges are the two that depend on which columns hold a tenant-wide + stratum, and resolving that costs another document read on a hot path. `origin` and `owner` do + not depend on it at all. ⛔ SO NEITHER `values` NOR `audience` MAY BE EMITTED FROM THIS CALL: + lending an empty key set makes both of them answer for a world in which nothing is shared. A + consumer that needs those two badges asks `field_classes` with the real set, the way + `routes_customers`' grid assembly does. + + ⚠ NO SESSION MEANS NO PROVENANCE, AND THAT IS HONEST DEGRADATION RATHER THAN A GAP. Two callers + arrive without one (`_fold_legacy_scope` on the live PUT path, `verify_api`'s W30a probe), and + the only reads that could answer them are TENANT-BLIND module-level ones: `shares.grants` with + no `st` opens the default repo's registry, which on any tenant but #0 is a different workspace's + grants attributed to this one. An unresolved answer costs those two callers nothing — neither + reads provenance — and a wrong one would be a tenant leak. `origin` is still answered, because + `field_origin` is a pure read of the definition in hand and touches no store. + """ + try: + from core import field_permissions as _fp + except Exception: # noqa: BLE001 + return {} + defs = [f for f in src if isinstance(f, dict) and f.get("key")] + classes = {} + admins = None + if session is not None: + admins = _admin_usernames() + try: + # ⛔ `grant_topic` IS THE MODULE KEY, NOT THE STORE BUCKET. `shares.field_oid` namespaces + # a grant by the topic the write doors already use — `"customer_data"` / + # `"product_data"` (`routes_customers`/`routes_products`) and the table key itself for a + # `ut_*` database (`routes_tables`) — which is `key` in every one of those three cases. + # Handing it `customer_table_workspace` finds no record for any column and every owner + # silently reads as absent, which is the `grant_records` docstring's own warning. + classes = _fp.field_classes(defs, getattr(session, "uname", ""), + grant_topic=key, values_shared=(), + st=getattr(session, "runtime", None)) + except Exception: # noqa: BLE001 + # A registry that will not open costs the OWNER, not the picker. The rows still ship, + # still carry every membership boolean, and say `None` about what they could not read. + classes = {} + out = {} + for f in defs: + fkey = f["key"] + bag = classes.get(fkey) or {} + owner = bag.get("owner") or None + try: + origin = _fp.field_origin(f) + except Exception: # noqa: BLE001 + origin = "preset" + out[fkey] = {"createdBy": (str(f.get("createdBy") or "").strip().lower() or None), + "owner": owner, + "ownerIsAdmin": (None if not owner or admins is None + else owner in admins), + "origin": origin} + return out + + def _module_fields(key, session=None): """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, so an admin filters on exactly what the user sees. @@ -679,7 +876,31 @@ def _module_fields(key, session=None): # skipped on a key collision so a declared column always wins its own key. taken = {f["key"] for f in out} out.extend(m for m in _metric_fields(key) if m["key"] not in taken) - return out + # ⭐⭐ W41-T08 / R8 / C1 — PROVENANCE, STAMPED AS A LAST PASS OVER THE FINISHED LIST. + # ⛔ A PASS, NOT A FILTER, AND THE POSITION IS THE PROOF. Every row that reached `out` above + # leaves this function, in the same order, with the same key/label/type/options/pinned and the + # same four membership booleans; the only difference is four ADDED keys. T09 arms R8's + # admin-only rule and it is blocked on the census this feeds — narrowing the offer here would + # build T09 by accident and break `Manual allocation of agent` the day after D-470 made it work. + # ⚠ A NEW DICT PER ROW, NEVER AN IN-PLACE STAMP — the rule `routes_customers`' grid assembly + # states over its own `class` bag, for the same reason. `owner` and `ownerIsAdmin` are THIS + # request's answer about a roster that changes, so stamping a row this function did not build + # itself (every `_metric_fields` row) would publish that answer to whatever else holds it. + # Those rows are freshly built today; the copy makes the aliasing impossible to reintroduce + # rather than merely untrue now. + # ⚠ A ROW WITH NO DEFINITION BEHIND IT (every `measure_` pseudo-field) takes + # `_PROVENANCE_UNKNOWN`, whose `origin` is `"preset"` — see the note there. + # ⭐⭐ W41-T09 / R8 — AND THE RULE IS APPLIED IN THE SAME PASS, ON THE STAMPED ROW. + # ⛔ THE ORDER IS THE POINT: the provenance has to be ON the row before anything can read + # `ownerIsAdmin` off it, and the rule reads nothing else — no second store call, no second + # derivation of who owns what. The list is still KEY-SET PRESERVING and still in the same + # order; a refused column keeps its row, its label and its four membership booleans, and + # loses exactly one flag while gaining one sentence. That is what keeps `hiddenFields` + # whole while the FILTER narrows, and it is why the picker and the validator below can + # still be the one vocabulary this function's docstring promises. + prov = _field_provenance(src, key, session) + return [_apply_admin_only_filter_rule(dict(r, **prov.get(r["key"], _PROVENANCE_UNKNOWN))) + for r in out] def _metric_fields(key): @@ -1218,6 +1439,19 @@ def _clean_perms(v, governed_keys=None, session=None): # per-user private one cannot, and those keep denying every leaf they are named in. row_wall_blind = _row_wall_blind_keys(key, session) filter_keys -= row_wall_blind + # ⭐⭐ W41-T09 / R8 — THE COLUMNS THE ADMIN-OWNER RULE TOOK OUT OF THE BUILDER, READ OFF + # THE ROWS THE PICKER WAS BUILT FROM rather than recomputed here. + # ⛔ THIS IS THE ONE-SOURCE INVARIANT DOING ITS JOB, not a convenience. Re-deriving "does + # this owner hold admin" in the validator would be a SECOND registry read at a SECOND + # moment, and the two would disagree the first time a role changed between the GET and + # the PUT: the admin would be refused a column the screen in front of them was still + # offering, or offered one the wall then rejected. `_apply_admin_only_filter_rule` already + # decided, on this very call, and the decision travels on the row. + # ⚠ `filter_keys` ALREADY EXCLUDES THESE (the rule cleared `filterable`), so this set is + # not what refuses them — the generic leaf-count check below would do that on its own, + # with the "unknown field" wording. It exists so the refusal can NAME THE COLUMN and give + # the cause, which is the half of done-when the generic message cannot carry. + admin_only_blocked = {f["key"] for f in fields_here if f.get(_FILTER_REASON_KEY)} hidden = raw.get("hiddenFields") or [] if not isinstance(hidden, list): raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") @@ -1302,6 +1536,28 @@ def _clean_perms(v, governed_keys=None, session=None): f"would hide EVERY row with nothing on screen saying why. Hide the " f"column instead, or filter on one of the database's own columns " f"or on a column shared with the whole workspace.") + # ⭐⭐ W41-T09 / R8 / OWNER INSTRUCTION 15 — THE WRITE DOOR, AND IT NAMES THE + # COLUMN. Second rather than first on purpose: a column that is BOTH unwallable + # and non-admin-owned gets the message above, because reassigning its owner would + # not make it filterable and this one would send the admin to do exactly that. + # ⛔ AND IT IS NOT DECORATION OVER THE GENERIC CHECK BELOW. Without it the same + # submission still 400s, worded *"an unknown field"* about a column the picker + # listed one request ago and the grid renders every day. That wording sends an + # administrator to look for a typo, and they will simply try again; owner + # instruction 15 is a RULE, and a rule that cannot say its own name is + # indistinguishable from a bug [[a-declared-gate-is-an-unchecked-claim]]. + # ⚠ BOTH DOORS ARE COVERED BY THIS ONE BRANCH: `put_perms` and + # `routes_slack.put_channel_agent_perms` call THIS validator, both threading a + # session, so a channel agent cannot be walled on a column a person cannot be. + denied = sorted(_leaf_col_ids(nodes) & admin_only_blocked) + if denied: + raise err(400, "field_owner_not_admin", + f"{key}: a permission filter cannot use {denied}. A permanent rule " + f"may only name a column that an administrator owns, because " + f"deleting a column also deletes the permission filter naming it, " + f"and these columns belong to accounts without the admin role. " + f"Reassign the column to an administrator, or write the rule on one " + f"of the database's own columns.") cleaned = aios_grid.clean_filter_tree(nodes, filter_keys, cohort_ids=None) if _leaf_count(cleaned) != _leaf_count(nodes): raise err(400, "bad_filter", diff --git a/api/routes_agent_harness.py b/api/routes_agent_harness.py index ed34831c623af46ef28bd312848275948108c70b..cfe1ffa6958ea00035eaaaf64d5c826e246fb209 100644 --- a/api/routes_agent_harness.py +++ b/api/routes_agent_harness.py @@ -1,455 +1,455 @@ -"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. - -Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file -pertaining to the agent skills, router etc. So we build a user can build a custom harness for each -agent through the chat interface."* - - GET /api/v1/agents/{id}/harness the file LIST (no bodies) - GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions - GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim - PUT /api/v1/agents/{id}/harness write a NEW version of one file - DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history - -⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write -appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the -history is a record of what happened rather than of what somebody last wanted it to look like. - -⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, -`routes_automation`); a harness file is not. Do not copy this file's posture over there or that -one's over here — the two rulings differ on purpose and they sit one screen apart in the product. - -⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every -write through this router is made BY a signed-in administrator, so `author` is stamped from the -session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by -a person or by the agent in the chat — a claim the client is entitled to make, because both are -permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side -door the automation engine uses when the agent writes with no session at all; that one stamps the -agent's own id as the author, which is the only case where `author` is not a username. - -⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of -one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in -`trimmed` on every payload that mentions the file, so a reader can see that the history is partial -rather than infer that the file was only ever saved twice. This is the tenant document, which is -already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is -a store-sized leak with a UI in front of it. -""" -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides -#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule -#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. -HARNESS_KEY = "agent_harness" - -#: One body. Generous for a skill or a router file and far under the point where a single write -#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a -#: truncation: a silently truncated skill file is a harness that does not do what its text says. -MAX_BODY_BYTES = 128 * 1024 - -#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving -#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. -MAX_FILES = 64 - -#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). -MAX_VERSIONS = 100 - -MAX_PATH = 200 - -#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — -#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is -#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a -#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a -#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. -_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") - -AUTHOR_KINDS = ("user", "agent") - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must - degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" - try: - found = rt.get(HARNESS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _files(rt, agent_id): - found = _all(rt).get(str(agent_id)) - return found if isinstance(found, dict) else {} - - -#: What kind of principal an id names. `None` = this tenant has no agent with that id. -#: -#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY -#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under -#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — -#: `Shell.tsx` mounts ``, and -#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds -#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that -#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival -#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on -#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO -#: occurrences of Canvas, Properties, Run history or panelTabs. -#: -#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent -#: skills, router" files in two places is the parallel code path item 13 exists to refuse. -AGENT_SLACK = "slack" -AGENT_AUTOMATION = "automation" - - -def agent_kind(rt, agent_id): - """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. - - ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at - read time from a column definition or a connector schedule; they have no stored home, so a - harness file hung off one is an orphan the moment the column changes. `patch_automation` - already refuses those ids for the neighbouring reason, and this refuses them by simply not - finding them — `all_definitions` holds stored automations only. - """ - import routes_slack - aid = str(agent_id or "") - if isinstance(routes_slack._agents(rt).get(aid), dict): - return AGENT_SLACK - import automation_engine as engine - try: - known = engine.all_definitions(rt) or {} - except Exception: # noqa: BLE001 - return None - return AGENT_AUTOMATION if aid in known else None - - -def _known_agent(rt, agent_id): - """Does this tenant have an agent with this id, in EITHER registry?""" - return agent_kind(rt, agent_id) is not None - - -def agent_wall(session, agent_id): - """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. - - ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a - THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door - in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in - `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the - agent it hangs off, so it is reached by whoever may already configure that agent. Picking one - of the two walls for both would either lock the Agents module's own users out of a tab the - owner asked for, or hand the Slack permission wall to anyone with an automation grant. - """ - kind = agent_kind(session.runtime, agent_id) - if kind is None: - raise err(404, "no_agent", "there is no agent with that id in this workspace") - if kind == AGENT_SLACK: - import core.perms as perms - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - else: - session.require("automation") - return kind - - -def normalize_path(raw): - """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. - - ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There - are two ways into this store — the HTTP route (which must answer 400) and `record_version()` - (which must raise `ValueError`, having no response to put a status into). Written twice, the - two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one - copy is invisible because the other still refuses, so nothing goes red and the wall is now - half there. Written once, a change to the rule is felt at both doors and by the gate. - """ - path = str(raw or "").strip().strip("/") - if not path or len(path) > MAX_PATH: - return None - if set(path) - _PATH_OK or ".." in path or "//" in path: - return None - return path - - -def _clean_path(raw): - """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" - path = normalize_path(raw) - if path is None: - if not str(raw or "").strip().strip("/"): - raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") - if len(str(raw)) > MAX_PATH: - raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") - raise err(400, "bad_path", - "a harness path may use letters, digits, dot, dash, underscore and / only") - return path - - -def _blank(path, author, author_kind): - return {"path": path, "versions": [], "trimmed": 0, - "created": _now(), "createdBy": author, "createdKind": author_kind} - - -def _append(record, body, author, author_kind, restored_from=None): - """Append ONE version to a file record, in place, and report what was trimmed. - - The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. - A version list whose numbers restart at 1 after a trim would make two different bodies share a - name, and `restoredFrom` would then point at whichever one happened to be in the window. - """ - # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback - # the live document and may run it more than once; appending in place would then stack two - # copies of the same version into the history on a retry. - prior = record.get("versions") if isinstance(record.get("versions"), list) else [] - last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) - entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, - "created": _now(), "bytes": len(body.encode("utf-8"))} - if restored_from: - entry["restoredFrom"] = int(restored_from) - versions = [*prior, entry] - dropped = max(0, len(versions) - MAX_VERSIONS) - if dropped: - versions = versions[dropped:] - record["versions"] = versions - record["trimmed"] = int(record.get("trimmed") or 0) + dropped - return entry - - -def _head(record): - """The newest version of a file record, or `None` for a record with no versions at all.""" - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - return versions[-1] if versions else None - - -def _row(record): - """One file, as the LIST door reports it: everything except the bodies. - - ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would - ship the whole harness on every pane render; the Harness tab lists first and opens one file - second, which is exactly the shape this answers. - """ - head = _head(record) or {} - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - return {"path": record.get("path") or "", - "version": int(head.get("version") or 0), - "bytes": int(head.get("bytes") or 0), - "author": head.get("author") or record.get("createdBy") or "", - "authorKind": head.get("authorKind") or record.get("createdKind") or "user", - "updated": head.get("created") or record.get("created") or "", - "created": record.get("created") or "", - "versions": len(versions), - "trimmed": int(record.get("trimmed") or 0)} - - -def _version_rows(record): - """The history of one file, newest first, WITHOUT the bodies. - - A body per version is what makes a diff possible, and it is also what makes this payload big: - 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing - (`?path=…&version=N`), which is two round trips for a diff and none for a history list. - """ - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - out = [] - for entry in reversed(versions): - if not isinstance(entry, dict): - continue - row = {"version": int(entry.get("version") or 0), - "author": entry.get("author") or "", - "authorKind": entry.get("authorKind") or "user", - "created": entry.get("created") or "", - "bytes": int(entry.get("bytes") or 0)} - if entry.get("restoredFrom"): - row["restoredFrom"] = int(entry["restoredFrom"]) - out.append(row) - return out - - -def _limits(): - """The caps, IN the payload, so a client can say "this file is full" before a write fails. - - ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the - one cap that acts without refusing; these report the three that refuse. - """ - return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, - "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} - - -# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── -def record_version(runtime, agent_id, path, body, author="", author_kind="agent", - restored_from=None): - """Write one version from INSIDE the server — the agent's own half of R8. - - ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" - true. An agent acting inside an automation run holds no session and no cookie; if its only way - to write were the HTTP door it would have to borrow a person's identity, and the authorship - column would then be a record of who was logged in rather than of what wrote the file. - - Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a - `ValueError`, because a server-side caller has no HTTP response to put a 404 into. - """ - agent_id = str(agent_id or "") - if not _known_agent(runtime, agent_id): - raise ValueError(f"no agent {agent_id!r} in this workspace") - path = normalize_path(path) - if path is None: - raise ValueError("that is not an acceptable harness path") - body = str(body or "") - if len(body.encode("utf-8")) > MAX_BODY_BYTES: - raise ValueError("harness body is over the size limit") - kind = author_kind if author_kind in AUTHOR_KINDS else "agent" - author = str(author or agent_id) - # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a - # store `update` callback propagates out of a half-run read-modify-write, and the one thing a - # refusal must never do is leave the caller unsure whether the write happened. - existing = _files(runtime, agent_id) - if path not in existing and len(existing) >= MAX_FILES: - raise ValueError(f"this agent already has {MAX_FILES} harness files") - appended = {} - - def _set(cur): - cur = dict(cur or {}) - files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} - record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) - appended.clear() - appended.update(_append(record, body, author, kind, restored_from)) - files[path] = record - cur[agent_id] = files - return cur - - runtime.update(HARNESS_KEY, _set, flush="sync") - return appended - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/agents/{agent_id}/harness") -def get_harness(agent_id: str, path: str = "", version: int = 0, - session: Session = Depends(require_session)): - """The list, one file, or one older body — decided by the query string (C4). - - ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than - consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to - editing a permission than to editing a document. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - files = _files(session.runtime, agent_id) - if not path: - rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] - return {"agent": agent_id, "files": rows, "limits": _limits()} - - wanted = _clean_path(path) - record = files.get(wanted) - if not isinstance(record, dict): - raise err(404, "no_file", f"this agent has no harness file at {wanted}") - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - if version: - for entry in versions: - if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): - return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", - "atVersion": int(version), "history": _version_rows(record), - "limits": _limits()} - # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked - # for something that existed and no longer does, and telling it apart from a bad number is - # the difference between "roll back to v3" failing loudly and failing as if v3 never was. - trimmed = int(record.get("trimmed") or 0) - if trimmed and int(version) <= trimmed: - raise err(410, "version_trimmed", - f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " - f"for this file, and its body is gone") - raise err(404, "no_version", f"this file has no version {int(version)}") - head = _head(record) or {} - return {"agent": agent_id, **_row(record), "body": head.get("body") or "", - "atVersion": int(head.get("version") or 0), "history": _version_rows(record), - "limits": _limits()} - - -@router.put("/agents/{agent_id}/harness") -def put_harness(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). - - ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a - UI affordance; it is this function refusing to have a code path that replaces a body. A - roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that - somebody went back rather than pretending the intervening versions never happened. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - body = body if isinstance(body, dict) else {} - path = _clean_path(body.get("path")) - text = body.get("body") - if not isinstance(text, str): - raise err(400, "no_body", "a harness file needs a body, even an empty one") - if len(text.encode("utf-8")) > MAX_BODY_BYTES: - raise err(413, "body_too_long", - f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") - - # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a - # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the - # session and is never read off the request. Nothing is bought by forging the first — both - # principals may write (R8) — and everything would be bought by forging the second. - kind = str(body.get("authorKind") or "user").strip().lower() - if kind not in AUTHOR_KINDS: - raise err(400, "bad_author_kind", "authorKind is either user or agent") - restored = body.get("restoredFrom") - try: - restored = int(restored) if restored else None - except (TypeError, ValueError): - raise err(400, "bad_version", "restoredFrom must be a version number") - - files = _files(session.runtime, agent_id) - if path not in files and len(files) >= MAX_FILES: - raise err(409, "too_many_files", - f"this agent already has {MAX_FILES} harness files; delete one to add another") - try: - record_version(session.runtime, agent_id, path, text, - author=session.uname, author_kind=kind, restored_from=restored) - except ValueError as exc: - raise err(400, "refused", str(exc)) - - fresh = _files(session.runtime, agent_id).get(path) - if not isinstance(fresh, dict) or not _head(fresh): - # The store took the write and did not record it. A 200 here would tell an administrator - # their skill file was saved when it was not — the shape `routes_slack` refuses too. - raise err(503, "store_unavailable", "the harness file was NOT saved") - head = _head(fresh) - return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", - "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), - "limits": _limits()} - - -@router.delete("/agents/{agent_id}/harness") -def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): - """Drop one harness file AND its history. - - ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing - versions as a side effect of an edit. Deleting a file is a person deciding the file should not - exist, which is a different act with a different button, and a store with no way to remove a - file is one where a typo'd path is permanent. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - wanted = _clean_path(path) - if wanted not in _files(session.runtime, agent_id): - raise err(404, "no_file", f"this agent has no harness file at {wanted}") - - def _set(cur): - cur = dict(cur or {}) - files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} - files.pop(wanted, None) - # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a - # bucket accumulates a row for every agent anybody ever opened the tab on. - if files: - cur[agent_id] = files - else: - cur.pop(agent_id, None) - return cur - - session.runtime.update(HARNESS_KEY, _set, flush="sync") - return {"agent": agent_id, "deleted": wanted, - "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) - if isinstance(rec, dict)], - "limits": _limits()} +"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. + +Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file +pertaining to the agent skills, router etc. So we build a user can build a custom harness for each +agent through the chat interface."* + + GET /api/v1/agents/{id}/harness the file LIST (no bodies) + GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions + GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim + PUT /api/v1/agents/{id}/harness write a NEW version of one file + DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history + +⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write +appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the +history is a record of what happened rather than of what somebody last wanted it to look like. + +⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, +`routes_automation`); a harness file is not. Do not copy this file's posture over there or that +one's over here — the two rulings differ on purpose and they sit one screen apart in the product. + +⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every +write through this router is made BY a signed-in administrator, so `author` is stamped from the +session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by +a person or by the agent in the chat — a claim the client is entitled to make, because both are +permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side +door the automation engine uses when the agent writes with no session at all; that one stamps the +agent's own id as the author, which is the only case where `author` is not a username. + +⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of +one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in +`trimmed` on every payload that mentions the file, so a reader can see that the history is partial +rather than infer that the file was only ever saved twice. This is the tenant document, which is +already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is +a store-sized leak with a UI in front of it. +""" +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule +#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. +HARNESS_KEY = "agent_harness" + +#: One body. Generous for a skill or a router file and far under the point where a single write +#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a +#: truncation: a silently truncated skill file is a harness that does not do what its text says. +MAX_BODY_BYTES = 128 * 1024 + +#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving +#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. +MAX_FILES = 64 + +#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). +MAX_VERSIONS = 100 + +MAX_PATH = 200 + +#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — +#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is +#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a +#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a +#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. +_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") + +AUTHOR_KINDS = ("user", "agent") + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must + degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" + try: + found = rt.get(HARNESS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _files(rt, agent_id): + found = _all(rt).get(str(agent_id)) + return found if isinstance(found, dict) else {} + + +#: What kind of principal an id names. `None` = this tenant has no agent with that id. +#: +#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY +#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under +#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — +#: `Shell.tsx` mounts ``, and +#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds +#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that +#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival +#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on +#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO +#: occurrences of Canvas, Properties, Run history or panelTabs. +#: +#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent +#: skills, router" files in two places is the parallel code path item 13 exists to refuse. +AGENT_SLACK = "slack" +AGENT_AUTOMATION = "automation" + + +def agent_kind(rt, agent_id): + """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. + + ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at + read time from a column definition or a connector schedule; they have no stored home, so a + harness file hung off one is an orphan the moment the column changes. `patch_automation` + already refuses those ids for the neighbouring reason, and this refuses them by simply not + finding them — `all_definitions` holds stored automations only. + """ + import routes_slack + aid = str(agent_id or "") + if isinstance(routes_slack._agents(rt).get(aid), dict): + return AGENT_SLACK + import automation_engine as engine + try: + known = engine.all_definitions(rt) or {} + except Exception: # noqa: BLE001 + return None + return AGENT_AUTOMATION if aid in known else None + + +def _known_agent(rt, agent_id): + """Does this tenant have an agent with this id, in EITHER registry?""" + return agent_kind(rt, agent_id) is not None + + +def agent_wall(session, agent_id): + """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. + + ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a + THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door + in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in + `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the + agent it hangs off, so it is reached by whoever may already configure that agent. Picking one + of the two walls for both would either lock the Agents module's own users out of a tab the + owner asked for, or hand the Slack permission wall to anyone with an automation grant. + """ + kind = agent_kind(session.runtime, agent_id) + if kind is None: + raise err(404, "no_agent", "there is no agent with that id in this workspace") + if kind == AGENT_SLACK: + import core.perms as perms + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + else: + session.require("automation") + return kind + + +def normalize_path(raw): + """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. + + ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There + are two ways into this store — the HTTP route (which must answer 400) and `record_version()` + (which must raise `ValueError`, having no response to put a status into). Written twice, the + two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one + copy is invisible because the other still refuses, so nothing goes red and the wall is now + half there. Written once, a change to the rule is felt at both doors and by the gate. + """ + path = str(raw or "").strip().strip("/") + if not path or len(path) > MAX_PATH: + return None + if set(path) - _PATH_OK or ".." in path or "//" in path: + return None + return path + + +def _clean_path(raw): + """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" + path = normalize_path(raw) + if path is None: + if not str(raw or "").strip().strip("/"): + raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") + if len(str(raw)) > MAX_PATH: + raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") + raise err(400, "bad_path", + "a harness path may use letters, digits, dot, dash, underscore and / only") + return path + + +def _blank(path, author, author_kind): + return {"path": path, "versions": [], "trimmed": 0, + "created": _now(), "createdBy": author, "createdKind": author_kind} + + +def _append(record, body, author, author_kind, restored_from=None): + """Append ONE version to a file record, in place, and report what was trimmed. + + The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. + A version list whose numbers restart at 1 after a trim would make two different bodies share a + name, and `restoredFrom` would then point at whichever one happened to be in the window. + """ + # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback + # the live document and may run it more than once; appending in place would then stack two + # copies of the same version into the history on a retry. + prior = record.get("versions") if isinstance(record.get("versions"), list) else [] + last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) + entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, + "created": _now(), "bytes": len(body.encode("utf-8"))} + if restored_from: + entry["restoredFrom"] = int(restored_from) + versions = [*prior, entry] + dropped = max(0, len(versions) - MAX_VERSIONS) + if dropped: + versions = versions[dropped:] + record["versions"] = versions + record["trimmed"] = int(record.get("trimmed") or 0) + dropped + return entry + + +def _head(record): + """The newest version of a file record, or `None` for a record with no versions at all.""" + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return versions[-1] if versions else None + + +def _row(record): + """One file, as the LIST door reports it: everything except the bodies. + + ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would + ship the whole harness on every pane render; the Harness tab lists first and opens one file + second, which is exactly the shape this answers. + """ + head = _head(record) or {} + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return {"path": record.get("path") or "", + "version": int(head.get("version") or 0), + "bytes": int(head.get("bytes") or 0), + "author": head.get("author") or record.get("createdBy") or "", + "authorKind": head.get("authorKind") or record.get("createdKind") or "user", + "updated": head.get("created") or record.get("created") or "", + "created": record.get("created") or "", + "versions": len(versions), + "trimmed": int(record.get("trimmed") or 0)} + + +def _version_rows(record): + """The history of one file, newest first, WITHOUT the bodies. + + A body per version is what makes a diff possible, and it is also what makes this payload big: + 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing + (`?path=…&version=N`), which is two round trips for a diff and none for a history list. + """ + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + out = [] + for entry in reversed(versions): + if not isinstance(entry, dict): + continue + row = {"version": int(entry.get("version") or 0), + "author": entry.get("author") or "", + "authorKind": entry.get("authorKind") or "user", + "created": entry.get("created") or "", + "bytes": int(entry.get("bytes") or 0)} + if entry.get("restoredFrom"): + row["restoredFrom"] = int(entry["restoredFrom"]) + out.append(row) + return out + + +def _limits(): + """The caps, IN the payload, so a client can say "this file is full" before a write fails. + + ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the + one cap that acts without refusing; these report the three that refuse. + """ + return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, + "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} + + +# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── +def record_version(runtime, agent_id, path, body, author="", author_kind="agent", + restored_from=None): + """Write one version from INSIDE the server — the agent's own half of R8. + + ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" + true. An agent acting inside an automation run holds no session and no cookie; if its only way + to write were the HTTP door it would have to borrow a person's identity, and the authorship + column would then be a record of who was logged in rather than of what wrote the file. + + Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a + `ValueError`, because a server-side caller has no HTTP response to put a 404 into. + """ + agent_id = str(agent_id or "") + if not _known_agent(runtime, agent_id): + raise ValueError(f"no agent {agent_id!r} in this workspace") + path = normalize_path(path) + if path is None: + raise ValueError("that is not an acceptable harness path") + body = str(body or "") + if len(body.encode("utf-8")) > MAX_BODY_BYTES: + raise ValueError("harness body is over the size limit") + kind = author_kind if author_kind in AUTHOR_KINDS else "agent" + author = str(author or agent_id) + # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a + # store `update` callback propagates out of a half-run read-modify-write, and the one thing a + # refusal must never do is leave the caller unsure whether the write happened. + existing = _files(runtime, agent_id) + if path not in existing and len(existing) >= MAX_FILES: + raise ValueError(f"this agent already has {MAX_FILES} harness files") + appended = {} + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) + appended.clear() + appended.update(_append(record, body, author, kind, restored_from)) + files[path] = record + cur[agent_id] = files + return cur + + runtime.update(HARNESS_KEY, _set, flush="sync") + return appended + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/agents/{agent_id}/harness") +def get_harness(agent_id: str, path: str = "", version: int = 0, + session: Session = Depends(require_session)): + """The list, one file, or one older body — decided by the query string (C4). + + ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than + consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to + editing a permission than to editing a document. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + files = _files(session.runtime, agent_id) + if not path: + rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] + return {"agent": agent_id, "files": rows, "limits": _limits()} + + wanted = _clean_path(path) + record = files.get(wanted) + if not isinstance(record, dict): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + if version: + for entry in versions: + if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): + return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", + "atVersion": int(version), "history": _version_rows(record), + "limits": _limits()} + # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked + # for something that existed and no longer does, and telling it apart from a bad number is + # the difference between "roll back to v3" failing loudly and failing as if v3 never was. + trimmed = int(record.get("trimmed") or 0) + if trimmed and int(version) <= trimmed: + raise err(410, "version_trimmed", + f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " + f"for this file, and its body is gone") + raise err(404, "no_version", f"this file has no version {int(version)}") + head = _head(record) or {} + return {"agent": agent_id, **_row(record), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(record), + "limits": _limits()} + + +@router.put("/agents/{agent_id}/harness") +def put_harness(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). + + ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a + UI affordance; it is this function refusing to have a code path that replaces a body. A + roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that + somebody went back rather than pretending the intervening versions never happened. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + body = body if isinstance(body, dict) else {} + path = _clean_path(body.get("path")) + text = body.get("body") + if not isinstance(text, str): + raise err(400, "no_body", "a harness file needs a body, even an empty one") + if len(text.encode("utf-8")) > MAX_BODY_BYTES: + raise err(413, "body_too_long", + f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") + + # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a + # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the + # session and is never read off the request. Nothing is bought by forging the first — both + # principals may write (R8) — and everything would be bought by forging the second. + kind = str(body.get("authorKind") or "user").strip().lower() + if kind not in AUTHOR_KINDS: + raise err(400, "bad_author_kind", "authorKind is either user or agent") + restored = body.get("restoredFrom") + try: + restored = int(restored) if restored else None + except (TypeError, ValueError): + raise err(400, "bad_version", "restoredFrom must be a version number") + + files = _files(session.runtime, agent_id) + if path not in files and len(files) >= MAX_FILES: + raise err(409, "too_many_files", + f"this agent already has {MAX_FILES} harness files; delete one to add another") + try: + record_version(session.runtime, agent_id, path, text, + author=session.uname, author_kind=kind, restored_from=restored) + except ValueError as exc: + raise err(400, "refused", str(exc)) + + fresh = _files(session.runtime, agent_id).get(path) + if not isinstance(fresh, dict) or not _head(fresh): + # The store took the write and did not record it. A 200 here would tell an administrator + # their skill file was saved when it was not — the shape `routes_slack` refuses too. + raise err(503, "store_unavailable", "the harness file was NOT saved") + head = _head(fresh) + return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), + "limits": _limits()} + + +@router.delete("/agents/{agent_id}/harness") +def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): + """Drop one harness file AND its history. + + ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing + versions as a side effect of an edit. Deleting a file is a person deciding the file should not + exist, which is a different act with a different button, and a store with no way to remove a + file is one where a typo'd path is permanent. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + wanted = _clean_path(path) + if wanted not in _files(session.runtime, agent_id): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + files.pop(wanted, None) + # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a + # bucket accumulates a row for every agent anybody ever opened the tab on. + if files: + cur[agent_id] = files + else: + cur.pop(agent_id, None) + return cur + + session.runtime.update(HARNESS_KEY, _set, flush="sync") + return {"agent": agent_id, "deleted": wanted, + "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) + if isinstance(rec, dict)], + "limits": _limits()} diff --git a/api/routes_alerts.py b/api/routes_alerts.py index 3ff47062772bd99f08e4cd0eb40d25e5e9382404..c651d88cf37431431a9588f38c540b71c89597df 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -1,668 +1,668 @@ -"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). - - GET /api/v1/alerts -> {alerts:[...]} - POST /api/v1/alerts <- {viewId, topic, label?} - DELETE /api/v1/alerts/{alert_id} - POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) - GET /api/v1/notifications -> {unread, items:[...]} - POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} - -The semantics — an alert is a view plus a remembered matched set, a notification is a NEW -ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. -This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. - -⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool -for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a -full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole -book, and the notification would name customers that user may not see — a permission leak wearing -a notification's clothes. The owner's own scope is the only correct basis for their alert. - -⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same -`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an -alert can see is by construction a row its owner could open. Re-implementing the filter here -would be a second definition of "matches", and those two would drift. -""" -import re - -from fastapi import APIRouter, Body, Depends - -import core.alerts as alerts -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. -_TOPICS = ("customer", "product") - -# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── -# -# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). -# -# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three -# keys onto the record at write time would give them to notifications minted AFTER the deploy and -# to nothing else — every notification already sitting in every tenant's inbox would open nothing, -# and the feature would be correct in the source and absent from the product -# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a -# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is -# another lane's file this wave — but that is the convenience, not the reason. -# -# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them -# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets -# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding -# here means the client branches on ONE field instead of re-deriving the same split. -# -# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` -# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by -# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) -# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, -# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review -# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real -# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored -# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary -# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed -# at was deleted two waves ago. - -#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them -#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). -NOTIF_KIND_ALERT = "alert" -NOTIF_KIND_AUTOMATION = "automation" -NOTIF_KIND_SHARE = "share" - -#: C3's `target.module` vocabulary, and the automation sub-selection. -TARGET_MODULE_DATABASE = "database" -TARGET_MODULE_AUTOMATION = "automation" -TARGET_TAB_RUNS = "runs" - -#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with -#: no producer is a string that reads as a feature — the reason this constant is named here and -#: cited from `routes_shares` rather than typed twice). -SHARE_TOPIC = "share" - -#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. -AUTOMATION_TOPIC = "automation" - -_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") - - -def route_for_topic(topic): - """A grid SCOPE key -> the registry route that renders it, or None. - - ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED - (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only - pair that differ — the registry names the surface (`customer_data`) while the grid names the - scope (`customer`) — so a topic passed through as a route sends every click to a page that - does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT - rather than plausible, because an absent target renders as a row that does not pretend to be - clickable, and a wrong one renders as a click that silently goes nowhere. - """ - t = str(topic or "").strip() - if t == "customer": - return "customer_data" - if t == "product": - return "product_data" - if _UT_TOPIC.match(t): - return t - return None - - -def _refusal_code(exc): - """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. - - ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` - (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own - cause. Anything that reduces all four to one word is throwing away the only information the - reader could have acted on. Returns `""` for a plain exception, so a caller can tell - "refused, and here is why" apart from "broke, and we do not know why". - """ - detail = getattr(exc, "detail", None) - if isinstance(detail, dict): - inner = detail.get("error") - if isinstance(inner, dict): - return str(inner.get("code") or "") - return "" - - -def notification_view(item): - """One STORED notification -> the shape the Inbox renders. PURE, and total. - - Never raises and never drops a row: an item it cannot classify comes back as an `alert` with - no `target`, which the client renders as an unclickable row rather than hiding. An inbox that - silently omits what it does not understand is the one failure a reader cannot detect. - """ - if not isinstance(item, dict): - return item - topic = str(item.get("topic") or "").strip() - alert_id = str(item.get("alertId") or "").strip() - - # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic - # says `automation` but whose producer key never arrived (a truncated payload, a server - # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work - # and silently does not, which is this repo's most-repeated failure shape. Failing the test - # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. - if topic == AUTOMATION_TOPIC and alert_id: - kind = NOTIF_KIND_AUTOMATION - target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} - elif topic == SHARE_TOPIC and alert_id: - # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, - # `row_id=`. - # - # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it - # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the - # target read `{module: "database", id: "view_42"}` — an instruction to open a database - # called `view_42`. It looked right in the payload and would have opened nothing. The - # producer resolves the object to its topic and hands over the route; this branch only - # shapes what it is given. - kind = NOTIF_KIND_SHARE - row_id = str(item.get("rowId") or "").strip() - target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, - **({"tab": row_id} if row_id else {})} - else: - kind = NOTIF_KIND_ALERT - route = route_for_topic(topic) - view_id = str(item.get("viewId") or "").strip() - target = None if route is None else ( - {"module": TARGET_MODULE_DATABASE, "id": route, - **({"tab": view_id} if view_id else {})}) - - # The email split: `subject` is the HEADER (what this is about — the alert, the automation, - # the database), `label` stays the BODY (what happened — the record that entered, the run - # summary). They were one field, which is why a notification read as a sentence with no - # sender and the pane could not be laid out like mail. - subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() - # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it - # is D-101's dead one; leaving it through would give the client two vocabularies for one - # question, which is the defect this wave's item 6 is about in a different file. - # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. - # - # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was - # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — - # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in - # the model or in the markup. Mail has a from. This is it. - # - # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. - # An alert firing and an automation landing rows have no person behind them; their honest - # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a - # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and - # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with - # you") — a sender recovered by regexing a sentence breaks the first time the sentence is - # reworded, and it would break silently, in the header. - # - # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an - # empty from column reads as a broken inbox rather than as an old notification. - actor = str(item.get("actor") or "").strip() - if kind == NOTIF_KIND_SHARE: - sender = actor or "A teammate" - elif kind == NOTIF_KIND_AUTOMATION: - # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's - # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` - # runs FIRST, so this server literal won and every actor-less automation notification - # showed the retired module name in the inbox's From column. - sender = actor or "Agents" - else: - sender = actor or "Alerts" - out = {**item, "read": bool(item.get("read")), "kind": kind, - "subject": subject or "Notification", "sender": sender} - if target is not None: - out["target"] = target - return out - - -def inbox_view(box): - """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. - - ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a - recount here would make the badge a function of whatever this page happened to include, which - is the exact defect `alertsModel.parseInbox`'s own header records from the other side. - """ - if not isinstance(box, dict): - return box - items = box.get("items") - if not isinstance(items, list): - return box - # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client - # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. - # - # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and - # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH - # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" - # against a browser clock would re-introduce the drift the offset exists to remove — a reader a - # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands - # now come from the same machine. - # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — - # the note two lines up records what happened last time only one of them was enriched. - return {**box, "now": alerts._now_iso(), - "items": [notification_view(n) for n in items]} - - -def _view_by_id(g, view_id): - """One saved view out of an assembly, by id. `None` when there is no such view. - - ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS - FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND - `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes - it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on - it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the - `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** - - ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In - `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare - FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our - side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above - the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical - line is swallowed by `/notifications`' `except Exception: continue`, so **every stored - view-alert was silently dropped from the Inbox** and nobody had anything to report at all. - One expression, one loud symptom and one silent one. - - ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site - be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. - - ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was - keyed `{id: view}` — which is precisely why the gate was green while production raised on - every call. The fixture is moving to the production shape in this same change, and tolerating - both here means a caller that legitimately holds one cannot resurrect the bug. - """ - want = str(view_id or "") - if not want: - return None - views = (g or {}).get("views") - if isinstance(views, dict): - found = views.get(want) - return found if isinstance(found, dict) else None - if not isinstance(views, list): - return None - for v in views: - if isinstance(v, dict) and str(v.get("id") or "") == want: - return v - return None - - -def _topic_or_400(raw): - topic = str(raw or "").strip().lower() - if topic.startswith("ut_") or topic in _TOPICS: - return topic - raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") - - -def _owner_session(session: Session, owner: str): - """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). - - ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an - owner session is built by swapping the `user` RECORD and letting both derive themselves. An - earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the - first write hook of the wave; the properties are the single definition of who a session is, - and going around them is how a session with an admin flag and a non-admin record exists. - - Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather - than evaluating as somebody else, which is the fail-closed direction. - """ - import core.users as users - - if str(owner) == str(session.uname): - return session - rec = (users.registry() or {}).get(str(owner)) - if not isinstance(rec, dict) or not rec.get("active", True): - return None - # `_public` is THE definition of what a session may know about its own account (never a hash - # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a - # second definition, and the one that leaks is always the copy. - return Session(tenant=session.tenant, user=users._public(str(owner), rec), - claims=session.claims, runtime=session.runtime) - - -def _evaluate(session: Session, rec: dict, assemblies=None): - """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. - - ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole - of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each - one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two - alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, - because each `_evaluate` was a closed call. - ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module - note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak - wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and - must never share an entry. Getting that key wrong is the one way this optimisation could leak. - ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: - they evaluate ONE alert and a memo for a single call is pure overhead. - """ - import aios_grid - from harness import filter_eval - - owner_sess = _owner_session(session, rec.get("owner")) - if owner_sess is None: - return {"skipped": "owner_unavailable"} - topic = str(rec.get("topic") or "") - memo_key = (topic, str(owner_sess.uname)) - g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None - if g is None: - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. - # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the - # one-shot field-name correction acks for every `ut_` topic that has an alert — - # taking them from the `/workspace` refresh that exists to show them to the person - # who made the edit. The customer branch below has always passed False; this one - # inherited a default nobody re-read. An inbox poll must never consume a one-shot. - g = ut_assembly(owner_sess, topic, - storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", - consume_corrections=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) - except Exception as e: # noqa: BLE001 - # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not - # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it - # keeps a blanket catch — but it now reports the refusal's OWN code where there is - # one. `type(e).__name__` said `HTTPException` for four different causes, and - # `lastError` is the only place a user ever learns why an alert stopped firing. - # - # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an - # evaluation genuinely needs the rows to run the filter over. So an alert on a - # read-through grid is created (T22) and then skips at evaluation with - # `window_required` naming why — which is D-184's remaining half, and it is a - # SENTENCE now rather than silence. - return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} - if isinstance(assemblies, dict): - assemblies[memo_key] = g - - view = _view_by_id(g, rec.get("viewId")) - if not isinstance(view, dict): - # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than - # deleting the alert: an alert that silently vanishes is indistinguishable from one that - # never fires, and the user cannot debug what is not there. - return {"skipped": "view_missing"} - - # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived - # and overlay values on a row. Evaluating a filter against raw pool dicts would silently - # never match any condition on a user-created or measure column. - # - # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND - # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. - # - # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows - # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner - # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: - # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell - # evaluated as BLANK. Measured on one assembly, same view, same rows: - # rows_src state='unpaid' / 'paid' - # _evaluate saw state='' / '' ⇐ every base cell blank - # the GRID saw state='unpaid' / 'paid' - # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` - # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it - # inverted.** End to end: a row whose value arrived by import, automation, paste or the create - # door never fired; only a value typed as a hand EDIT did. - # - # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; - # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); - # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns - # `skipped: window_required`). - # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it - # would let a stale definition value shadow an edit the user has just made — the same defect - # `table_rows`' own note records, arriving from the other side. - _ov = (g.get("ws") or {}).get("overlays") or {} - _merged = {} - for _r in g["rows_src"]: - _pid = str(_r.get("pid")) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _o = _ov.get(_pid) - if isinstance(_o, dict): - _cells.update(_o) - _merged[_pid] = _cells - rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, - derived=g.get("derived")) - config = view.get("config") or view - ctx = filter_eval.EvalCtx( - cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} - for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, - measure_sets=g.get("measure_sets") or {}, - today=g.get("today")) - pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, - member_pids=config.get("memberPids")) - labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} - return alerts.evaluate(rec.get("id"), [str(p) for p in pids], - labels=labels, partial=False, st=session.runtime) - - -@router.get("/alerts") -def list_alerts(session: Session = Depends(require_session)): - return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime)} - - -@router.post("/alerts") -def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): - body = body or {} - view_id = str(body.get("viewId") or "").strip() - if not view_id: - raise err(400, "bad_view", "an alert needs the id of the view it watches") - topic = _topic_or_400(body.get("topic")) - _require_filtered_view(session, topic, view_id) - import uuid - aid = f"al_{uuid.uuid4().hex[:12]}" - rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, - label=body.get("label") or "", st=session.runtime) - # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". - # Deferring this to the first write hook would mean the next edit announces the whole view. - outcome = _evaluate(session, rec) - return {"alert": {**rec, "seeded": True}, "first": outcome} - - -def _require_filtered_view(session: Session, topic: str, view_id: str): - """400 unless `view_id` exists on `topic` AND actually narrows something. - - ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one - that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" - (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an - entrant again — there is nothing left to enter. The owner's words are *"when a Record gets - into that Filter's criteria"*: no criteria, no alert, and said at creation rather than - discovered by never being notified. - - `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a - half-typed rule is not a filter, and this must agree with what actually narrows or it would - accept a view whose one rule the engine then ignores. - - ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and - the second one hid the first. - - (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is - `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a - read-through grid built the whole pool — and `scoped_pool` refuses that with - `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` - (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, - runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, - closed** — an alert on a read-through grid can now be made at all. - (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` - is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and - `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by - *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED - A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design - (a 5xx body is the server's internals), substituting *"Something went wrong on our - side."* — which is the owner's screenshot, word for word. A knowable cause returned as a - 5xx is invisible by construction, so re-wording the 503 could never have fixed this. - ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is - honest. What it may no longer do is catch a refusal that already knows its own name. - """ - from fastapi import HTTPException - - from harness import filter_eval - - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⚠ `consume_corrections=False` — the customer branch has always passed it and this - # one inherited a default nobody re-read. Creating an alert must not eat the one-shot - # field-name correction acks belonging to the `/workspace` refresh that exists to show - # them to the person who made the edit. Same defect `_evaluate`'s header records. - g = ut_assembly(session, topic, - storage_key=f"{session.tenant}:{topic}:{session.uname}", - consume_corrections=False, with_rows=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(session, scope=topic, consume_corrections=False) - except HTTPException: - raise # it already names its own cause - except Exception as e: # noqa: BLE001 - # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, - # the one path that reaches this branch is also the one path with nothing to debug from. - raise err(503, "unavailable", - f"the table could not be read ({type(e).__name__}) — try again in a moment") - view = _view_by_id(g, view_id) - if not isinstance(view, dict): - raise err(404, "no_view", "that view does not exist on this table") - nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) - - # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT - # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS - # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have - # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines - # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the - # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. - # - # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the - # fix was right, and the three reasons it hid are the most transferable thing in this file. - # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each - # naming themselves as the origin of the same screenshot are mutually exclusive, and the next - # reader believes whichever they meet first — which is why this correction sits above rather - # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead - # of the working tree. [[grep-output-is-not-source]] - # - # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, - # that this was owner item 17 — see the correction directly above). - # - # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other - # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF - # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. - # - # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the - # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. - # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever - # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that - # worked was the refusal path: "Alert me about new records" had never once created an alert - # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the - # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went - # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. - # - # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check - # arity until the line RUNS, and this line runs only on the success path of a feature whose - # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, - # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal - # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, - # thorough and honest about everything except the one path the feature exists for. - # - # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a - # column up in; building a second dict here would be a second answer to one question, which - # is this wave's other headline defect in a different file. Its leading underscore is a real - # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. - columns = filter_eval._columns_map(g.get("fields") or []) - - def _any_active(ns): - for n in ns or (): - if isinstance(n, dict) and isinstance(n.get("children"), list): - if _any_active(n["children"]): - return True - elif filter_eval.is_rule_active(n, columns): - return True - return False - - if not _any_active(nodes): - raise err(400, "no_filter", - "this view has no active filter, so no record can ever ENTER it — add a " - "condition to the view first, then create the alert") - - -@router.delete("/alerts/{alert_id}") -def delete_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - if str(rec.get("owner")) != str(session.uname) and not session.admin: - raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") - alerts.delete(alert_id, st=session.runtime) - return {"ok": True} - - -@router.post("/alerts/{alert_id}/run") -def run_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - return _evaluate(session, rec) - - -@router.get("/notifications") -def notifications(session: Session = Depends(require_session)): - """The inbox — RE-EVALUATED on read, which is a deliberate design choice. - - ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The - plan was a push hook: the automation engine calls `after_write` when it lands rows. But - `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be - evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside - a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity - construction that leaks scope. - - Pulling on read has none of that: the caller IS a session, the assemblies are already - scope-cached, and the user cannot observe the difference — an inbox is only ever read by - someone opening it. The cost is that a notification is minted when you LOOK rather than when - the row landed, so the `at` stamp is detection time, not arrival time. - - `after_write` stays exported for the day the engine can hand over a real identity. - - ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. - ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is - **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** - (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the - re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the - body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole - grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which - is the difference between "fine" and "three seconds per alert" the day somebody uses the - feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what - this does: every alert is still evaluated, against the same rows, in the same order. - """ - assemblies = {} - for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): - try: - _evaluate(session, rec, assemblies=assemblies) - except Exception: # noqa: BLE001 - continue # one bad alert must not empty the pane - # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before - # this wave carries a `target` too. See `notification_view`'s header for why it is derived. - return inbox_view(alerts.inbox(session.uname, st=session.runtime)) - - -@router.post("/notifications/read") -def read_notifications(body: dict = Body(default=None), - session: Session = Depends(require_session)): - body = body or {} - ids = body.get("ids") - if ids is not None and not isinstance(ids, list): - raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") - # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox - # module re-renders from it — an un-enriched answer here would strip `target` off every row - # the moment somebody marked one read, i.e. the feature would work until first use. - return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), - st=session.runtime)) - - -def after_write(session: Session, topic_key: str): - """THE WRITE HOOK — call after a write that could change what a view matches. - - Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation - upserts reach it the same way. It never raises: an alert evaluation failing must not fail the - edit that triggered it. - - ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes - one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. - ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the - push hook was resolved the other way). Booked rather than wired: minting a session inside the - engine's worker thread is the ad-hoc identity construction this file exists to avoid. - """ - try: - assemblies = {} - return alerts.after_write(topic_key, st=session.runtime, - runner=lambda rec: _evaluate(session, rec, - assemblies=assemblies)) - except Exception: # noqa: BLE001 - return {"evaluated": 0} +"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). + + GET /api/v1/alerts -> {alerts:[...]} + POST /api/v1/alerts <- {viewId, topic, label?} + DELETE /api/v1/alerts/{alert_id} + POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) + GET /api/v1/notifications -> {unread, items:[...]} + POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} + +The semantics — an alert is a view plus a remembered matched set, a notification is a NEW +ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. +This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. + +⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool +for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a +full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole +book, and the notification would name customers that user may not see — a permission leak wearing +a notification's clothes. The owner's own scope is the only correct basis for their alert. + +⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same +`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an +alert can see is by construction a row its owner could open. Re-implementing the filter here +would be a second definition of "matches", and those two would drift. +""" +import re + +from fastapi import APIRouter, Body, Depends + +import core.alerts as alerts +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. +_TOPICS = ("customer", "product") + +# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── +# +# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). +# +# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three +# keys onto the record at write time would give them to notifications minted AFTER the deploy and +# to nothing else — every notification already sitting in every tenant's inbox would open nothing, +# and the feature would be correct in the source and absent from the product +# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a +# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is +# another lane's file this wave — but that is the convenience, not the reason. +# +# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them +# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets +# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding +# here means the client branches on ONE field instead of re-deriving the same split. +# +# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` +# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by +# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) +# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, +# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review +# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real +# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored +# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary +# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed +# at was deleted two waves ago. + +#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them +#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). +NOTIF_KIND_ALERT = "alert" +NOTIF_KIND_AUTOMATION = "automation" +NOTIF_KIND_SHARE = "share" + +#: C3's `target.module` vocabulary, and the automation sub-selection. +TARGET_MODULE_DATABASE = "database" +TARGET_MODULE_AUTOMATION = "automation" +TARGET_TAB_RUNS = "runs" + +#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with +#: no producer is a string that reads as a feature — the reason this constant is named here and +#: cited from `routes_shares` rather than typed twice). +SHARE_TOPIC = "share" + +#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. +AUTOMATION_TOPIC = "automation" + +_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") + + +def route_for_topic(topic): + """A grid SCOPE key -> the registry route that renders it, or None. + + ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED + (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only + pair that differ — the registry names the surface (`customer_data`) while the grid names the + scope (`customer`) — so a topic passed through as a route sends every click to a page that + does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT + rather than plausible, because an absent target renders as a row that does not pretend to be + clickable, and a wrong one renders as a click that silently goes nowhere. + """ + t = str(topic or "").strip() + if t == "customer": + return "customer_data" + if t == "product": + return "product_data" + if _UT_TOPIC.match(t): + return t + return None + + +def _refusal_code(exc): + """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. + + ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` + (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own + cause. Anything that reduces all four to one word is throwing away the only information the + reader could have acted on. Returns `""` for a plain exception, so a caller can tell + "refused, and here is why" apart from "broke, and we do not know why". + """ + detail = getattr(exc, "detail", None) + if isinstance(detail, dict): + inner = detail.get("error") + if isinstance(inner, dict): + return str(inner.get("code") or "") + return "" + + +def notification_view(item): + """One STORED notification -> the shape the Inbox renders. PURE, and total. + + Never raises and never drops a row: an item it cannot classify comes back as an `alert` with + no `target`, which the client renders as an unclickable row rather than hiding. An inbox that + silently omits what it does not understand is the one failure a reader cannot detect. + """ + if not isinstance(item, dict): + return item + topic = str(item.get("topic") or "").strip() + alert_id = str(item.get("alertId") or "").strip() + + # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic + # says `automation` but whose producer key never arrived (a truncated payload, a server + # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work + # and silently does not, which is this repo's most-repeated failure shape. Failing the test + # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. + if topic == AUTOMATION_TOPIC and alert_id: + kind = NOTIF_KIND_AUTOMATION + target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} + elif topic == SHARE_TOPIC and alert_id: + # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, + # `row_id=`. + # + # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it + # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the + # target read `{module: "database", id: "view_42"}` — an instruction to open a database + # called `view_42`. It looked right in the payload and would have opened nothing. The + # producer resolves the object to its topic and hands over the route; this branch only + # shapes what it is given. + kind = NOTIF_KIND_SHARE + row_id = str(item.get("rowId") or "").strip() + target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, + **({"tab": row_id} if row_id else {})} + else: + kind = NOTIF_KIND_ALERT + route = route_for_topic(topic) + view_id = str(item.get("viewId") or "").strip() + target = None if route is None else ( + {"module": TARGET_MODULE_DATABASE, "id": route, + **({"tab": view_id} if view_id else {})}) + + # The email split: `subject` is the HEADER (what this is about — the alert, the automation, + # the database), `label` stays the BODY (what happened — the record that entered, the run + # summary). They were one field, which is why a notification read as a sentence with no + # sender and the pane could not be laid out like mail. + subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() + # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it + # is D-101's dead one; leaving it through would give the client two vocabularies for one + # question, which is the defect this wave's item 6 is about in a different file. + # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. + # + # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was + # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — + # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in + # the model or in the markup. Mail has a from. This is it. + # + # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. + # An alert firing and an automation landing rows have no person behind them; their honest + # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a + # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and + # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with + # you") — a sender recovered by regexing a sentence breaks the first time the sentence is + # reworded, and it would break silently, in the header. + # + # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an + # empty from column reads as a broken inbox rather than as an old notification. + actor = str(item.get("actor") or "").strip() + if kind == NOTIF_KIND_SHARE: + sender = actor or "A teammate" + elif kind == NOTIF_KIND_AUTOMATION: + # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's + # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` + # runs FIRST, so this server literal won and every actor-less automation notification + # showed the retired module name in the inbox's From column. + sender = actor or "Agents" + else: + sender = actor or "Alerts" + out = {**item, "read": bool(item.get("read")), "kind": kind, + "subject": subject or "Notification", "sender": sender} + if target is not None: + out["target"] = target + return out + + +def inbox_view(box): + """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. + + ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a + recount here would make the badge a function of whatever this page happened to include, which + is the exact defect `alertsModel.parseInbox`'s own header records from the other side. + """ + if not isinstance(box, dict): + return box + items = box.get("items") + if not isinstance(items, list): + return box + # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client + # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. + # + # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and + # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH + # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" + # against a browser clock would re-introduce the drift the offset exists to remove — a reader a + # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands + # now come from the same machine. + # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — + # the note two lines up records what happened last time only one of them was enriched. + return {**box, "now": alerts._now_iso(), + "items": [notification_view(n) for n in items]} + + +def _view_by_id(g, view_id): + """One saved view out of an assembly, by id. `None` when there is no such view. + + ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS + FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND + `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes + it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on + it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the + `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** + + ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In + `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare + FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our + side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above + the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical + line is swallowed by `/notifications`' `except Exception: continue`, so **every stored + view-alert was silently dropped from the Inbox** and nobody had anything to report at all. + One expression, one loud symptom and one silent one. + + ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site + be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. + + ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was + keyed `{id: view}` — which is precisely why the gate was green while production raised on + every call. The fixture is moving to the production shape in this same change, and tolerating + both here means a caller that legitimately holds one cannot resurrect the bug. + """ + want = str(view_id or "") + if not want: + return None + views = (g or {}).get("views") + if isinstance(views, dict): + found = views.get(want) + return found if isinstance(found, dict) else None + if not isinstance(views, list): + return None + for v in views: + if isinstance(v, dict) and str(v.get("id") or "") == want: + return v + return None + + +def _topic_or_400(raw): + topic = str(raw or "").strip().lower() + if topic.startswith("ut_") or topic in _TOPICS: + return topic + raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") + + +def _owner_session(session: Session, owner: str): + """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). + + ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an + owner session is built by swapping the `user` RECORD and letting both derive themselves. An + earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the + first write hook of the wave; the properties are the single definition of who a session is, + and going around them is how a session with an admin flag and a non-admin record exists. + + Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather + than evaluating as somebody else, which is the fail-closed direction. + """ + import core.users as users + + if str(owner) == str(session.uname): + return session + rec = (users.registry() or {}).get(str(owner)) + if not isinstance(rec, dict) or not rec.get("active", True): + return None + # `_public` is THE definition of what a session may know about its own account (never a hash + # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a + # second definition, and the one that leaks is always the copy. + return Session(tenant=session.tenant, user=users._public(str(owner), rec), + claims=session.claims, runtime=session.runtime) + + +def _evaluate(session: Session, rec: dict, assemblies=None): + """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. + + ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole + of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each + one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two + alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, + because each `_evaluate` was a closed call. + ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module + note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak + wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and + must never share an entry. Getting that key wrong is the one way this optimisation could leak. + ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: + they evaluate ONE alert and a memo for a single call is pure overhead. + """ + import aios_grid + from harness import filter_eval + + owner_sess = _owner_session(session, rec.get("owner")) + if owner_sess is None: + return {"skipped": "owner_unavailable"} + topic = str(rec.get("topic") or "") + memo_key = (topic, str(owner_sess.uname)) + g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None + if g is None: + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. + # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the + # one-shot field-name correction acks for every `ut_` topic that has an alert — + # taking them from the `/workspace` refresh that exists to show them to the person + # who made the edit. The customer branch below has always passed False; this one + # inherited a default nobody re-read. An inbox poll must never consume a one-shot. + g = ut_assembly(owner_sess, topic, + storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", + consume_corrections=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) + except Exception as e: # noqa: BLE001 + # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not + # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it + # keeps a blanket catch — but it now reports the refusal's OWN code where there is + # one. `type(e).__name__` said `HTTPException` for four different causes, and + # `lastError` is the only place a user ever learns why an alert stopped firing. + # + # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an + # evaluation genuinely needs the rows to run the filter over. So an alert on a + # read-through grid is created (T22) and then skips at evaluation with + # `window_required` naming why — which is D-184's remaining half, and it is a + # SENTENCE now rather than silence. + return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} + if isinstance(assemblies, dict): + assemblies[memo_key] = g + + view = _view_by_id(g, rec.get("viewId")) + if not isinstance(view, dict): + # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than + # deleting the alert: an alert that silently vanishes is indistinguishable from one that + # never fires, and the user cannot debug what is not there. + return {"skipped": "view_missing"} + + # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived + # and overlay values on a row. Evaluating a filter against raw pool dicts would silently + # never match any condition on a user-created or measure column. + # + # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND + # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. + # + # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows + # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner + # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: + # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell + # evaluated as BLANK. Measured on one assembly, same view, same rows: + # rows_src state='unpaid' / 'paid' + # _evaluate saw state='' / '' ⇐ every base cell blank + # the GRID saw state='unpaid' / 'paid' + # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` + # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it + # inverted.** End to end: a row whose value arrived by import, automation, paste or the create + # door never fired; only a value typed as a hand EDIT did. + # + # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; + # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); + # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns + # `skipped: window_required`). + # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it + # would let a stale definition value shadow an edit the user has just made — the same defect + # `table_rows`' own note records, arriving from the other side. + _ov = (g.get("ws") or {}).get("overlays") or {} + _merged = {} + for _r in g["rows_src"]: + _pid = str(_r.get("pid")) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _o = _ov.get(_pid) + if isinstance(_o, dict): + _cells.update(_o) + _merged[_pid] = _cells + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, + derived=g.get("derived")) + config = view.get("config") or view + ctx = filter_eval.EvalCtx( + cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} + for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, + measure_sets=g.get("measure_sets") or {}, + today=g.get("today")) + pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, + member_pids=config.get("memberPids")) + labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} + return alerts.evaluate(rec.get("id"), [str(p) for p in pids], + labels=labels, partial=False, st=session.runtime) + + +@router.get("/alerts") +def list_alerts(session: Session = Depends(require_session)): + return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime)} + + +@router.post("/alerts") +def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): + body = body or {} + view_id = str(body.get("viewId") or "").strip() + if not view_id: + raise err(400, "bad_view", "an alert needs the id of the view it watches") + topic = _topic_or_400(body.get("topic")) + _require_filtered_view(session, topic, view_id) + import uuid + aid = f"al_{uuid.uuid4().hex[:12]}" + rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, + label=body.get("label") or "", st=session.runtime) + # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". + # Deferring this to the first write hook would mean the next edit announces the whole view. + outcome = _evaluate(session, rec) + return {"alert": {**rec, "seeded": True}, "first": outcome} + + +def _require_filtered_view(session: Session, topic: str, view_id: str): + """400 unless `view_id` exists on `topic` AND actually narrows something. + + ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one + that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" + (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an + entrant again — there is nothing left to enter. The owner's words are *"when a Record gets + into that Filter's criteria"*: no criteria, no alert, and said at creation rather than + discovered by never being notified. + + `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a + half-typed rule is not a filter, and this must agree with what actually narrows or it would + accept a view whose one rule the engine then ignores. + + ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and + the second one hid the first. + + (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is + `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a + read-through grid built the whole pool — and `scoped_pool` refuses that with + `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` + (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, + runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, + closed** — an alert on a read-through grid can now be made at all. + (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` + is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and + `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by + *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED + A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design + (a 5xx body is the server's internals), substituting *"Something went wrong on our + side."* — which is the owner's screenshot, word for word. A knowable cause returned as a + 5xx is invisible by construction, so re-wording the 503 could never have fixed this. + ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is + honest. What it may no longer do is catch a refusal that already knows its own name. + """ + from fastapi import HTTPException + + from harness import filter_eval + + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⚠ `consume_corrections=False` — the customer branch has always passed it and this + # one inherited a default nobody re-read. Creating an alert must not eat the one-shot + # field-name correction acks belonging to the `/workspace` refresh that exists to show + # them to the person who made the edit. Same defect `_evaluate`'s header records. + g = ut_assembly(session, topic, + storage_key=f"{session.tenant}:{topic}:{session.uname}", + consume_corrections=False, with_rows=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(session, scope=topic, consume_corrections=False) + except HTTPException: + raise # it already names its own cause + except Exception as e: # noqa: BLE001 + # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, + # the one path that reaches this branch is also the one path with nothing to debug from. + raise err(503, "unavailable", + f"the table could not be read ({type(e).__name__}) — try again in a moment") + view = _view_by_id(g, view_id) + if not isinstance(view, dict): + raise err(404, "no_view", "that view does not exist on this table") + nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) + + # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT + # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS + # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have + # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines + # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the + # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. + # + # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the + # fix was right, and the three reasons it hid are the most transferable thing in this file. + # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each + # naming themselves as the origin of the same screenshot are mutually exclusive, and the next + # reader believes whichever they meet first — which is why this correction sits above rather + # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead + # of the working tree. [[grep-output-is-not-source]] + # + # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, + # that this was owner item 17 — see the correction directly above). + # + # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other + # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF + # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. + # + # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the + # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. + # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever + # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that + # worked was the refusal path: "Alert me about new records" had never once created an alert + # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the + # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went + # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. + # + # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check + # arity until the line RUNS, and this line runs only on the success path of a feature whose + # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, + # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal + # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, + # thorough and honest about everything except the one path the feature exists for. + # + # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a + # column up in; building a second dict here would be a second answer to one question, which + # is this wave's other headline defect in a different file. Its leading underscore is a real + # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. + columns = filter_eval._columns_map(g.get("fields") or []) + + def _any_active(ns): + for n in ns or (): + if isinstance(n, dict) and isinstance(n.get("children"), list): + if _any_active(n["children"]): + return True + elif filter_eval.is_rule_active(n, columns): + return True + return False + + if not _any_active(nodes): + raise err(400, "no_filter", + "this view has no active filter, so no record can ever ENTER it — add a " + "condition to the view first, then create the alert") + + +@router.delete("/alerts/{alert_id}") +def delete_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + if str(rec.get("owner")) != str(session.uname) and not session.admin: + raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") + alerts.delete(alert_id, st=session.runtime) + return {"ok": True} + + +@router.post("/alerts/{alert_id}/run") +def run_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + return _evaluate(session, rec) + + +@router.get("/notifications") +def notifications(session: Session = Depends(require_session)): + """The inbox — RE-EVALUATED on read, which is a deliberate design choice. + + ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The + plan was a push hook: the automation engine calls `after_write` when it lands rows. But + `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be + evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside + a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity + construction that leaks scope. + + Pulling on read has none of that: the caller IS a session, the assemblies are already + scope-cached, and the user cannot observe the difference — an inbox is only ever read by + someone opening it. The cost is that a notification is minted when you LOOK rather than when + the row landed, so the `at` stamp is detection time, not arrival time. + + `after_write` stays exported for the day the engine can hand over a real identity. + + ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. + ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is + **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** + (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the + re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the + body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole + grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which + is the difference between "fine" and "three seconds per alert" the day somebody uses the + feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what + this does: every alert is still evaluated, against the same rows, in the same order. + """ + assemblies = {} + for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): + try: + _evaluate(session, rec, assemblies=assemblies) + except Exception: # noqa: BLE001 + continue # one bad alert must not empty the pane + # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before + # this wave carries a `target` too. See `notification_view`'s header for why it is derived. + return inbox_view(alerts.inbox(session.uname, st=session.runtime)) + + +@router.post("/notifications/read") +def read_notifications(body: dict = Body(default=None), + session: Session = Depends(require_session)): + body = body or {} + ids = body.get("ids") + if ids is not None and not isinstance(ids, list): + raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") + # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox + # module re-renders from it — an un-enriched answer here would strip `target` off every row + # the moment somebody marked one read, i.e. the feature would work until first use. + return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), + st=session.runtime)) + + +def after_write(session: Session, topic_key: str): + """THE WRITE HOOK — call after a write that could change what a view matches. + + Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation + upserts reach it the same way. It never raises: an alert evaluation failing must not fail the + edit that triggered it. + + ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes + one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. + ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the + push hook was resolved the other way). Booked rather than wired: minting a session inside the + engine's worker thread is the ad-hoc identity construction this file exists to avoid. + """ + try: + assemblies = {} + return alerts.after_write(topic_key, st=session.runtime, + runner=lambda rec: _evaluate(session, rec, + assemblies=assemblies)) + except Exception: # noqa: BLE001 + return {"evaluated": 0} diff --git a/api/routes_customers.py b/api/routes_customers.py index e7fa633e5861d0c0962352cf2cd77d409a5996a6..91eecd8e20c9dbfe72879ba2806530497a45a1e0 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -1,1427 +1,1545 @@ -"""routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a). - -Two things happen here that did not happen in the pre-wave `main.py`: - - 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole - book, to anyone holding the shared APP_PASSWORD. Now the pool is built with - `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a - Fisch row and an agent-linked login never receives another rep's book. The scope is applied - at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist - in this response. - - 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home - for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and - whichever process you asked last was right. Reads and writes now both go through - `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3). - -⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is -cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API -container until its cache is refreshed, and vice versa. Deleting the fork removes the second -SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4, -owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient -proof is single-process and would report green on exactly the thing that is still broken. -""" -import math -import time - -from fastapi import APIRouter, Body, Depends - -import scope_cache -from deps import Session, err, module_gate, perms - -router = APIRouter(prefix="/api/v1") - -#: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a -#: 403 rather than an empty table that looks like "you have no customers". -MODULE = "customer_data" - -_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before - - -def _pool_rows(session: Session): - """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that - may be shared between users. - - ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is - `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is - genuinely scope-shaped — two users with the same BU and the same book are asking the same - question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation). - - The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped - and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_` - and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table - workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e. - PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one - would have been served the FIRST one's private notes and private columns. The scope key was - right for the pool and wrong for everything wrapped around it. - - It survived a green 129-check battery because every fixture user had a DISTINCT - `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not - express the bug. `verify_api.py` now carries a same-scope second user for exactly this. - """ - rt = session.runtime - team_id, agent = _team_agent(session) - return _pool_for(rt, team_id, agent) - - -def _pool_for(rt, team_id, agent): - """The cached pool for an explicit scope — session-free so the prewarm thread and the - stale-refresh path can call it. STALE-WHILE-REFRESH (scope_cache): once a copy exists no - request blocks on the 10–30s Odoo rebuild again; only a scope's FIRST-ever build does. - - DEBT-2 (2026-08-04): while the tenant's RESOLVED Odoo source is PAUSED this path never - goes live — it serves the in-process copy at any age, else the persisted pause-time - snapshot (restart-safe), else answers 503. Never a WIDER scope's snapshot: handing a - scoped user the consolidated rows would widen their book, which is worse than an error.""" - import modules.customer_data as cl - import routes_keychain - - key = ("pool", team_id, agent) - - if routes_keychain.odoo_paused(rt): - hit = rt.pool_cache.get(key) - if hit: - return hit[1] - snap = routes_keychain.load_pool_snapshot(rt, team_id, agent) - if snap is not None: - rt.pool_cache[key] = snap # seed, so the memo stamps stay coherent - return snap[1] - raise err(503, "connector_paused", - "this data source is paused and no snapshot exists for your scope — " - "an admin can resume it under Settings → Connectors") - - def _build(): - # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled - # builder the Streamlit page uses — passing the scope here is what makes the isolation a - # property of the QUERY instead of a filter someone can forget to apply downstream. - return cl.pool(agent, team_id) - - def _evict(): - # Bounded: a scope cache that only ever grows is a memory leak in a shared process. - if len(rt.pool_cache) > 16: - for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]: - rt.pool_cache.pop(stale, None) - - return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, _build, _evict) - - -def warm_default(rt): - """Boot prewarm: the consolidated pool `(None, None)` — the scope every admin and every - all-BU account lands on. Called from main.py's prewarm thread only.""" - _pool_for(rt, None, None) - - -def _team_agent(session: Session): - """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by - `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree. - - ⛔ WAVE 15 R1: THIS NOW COMES FROM THE PERMANENT FILTER (C-PERM amendment 3). `team_id` is - not a row filter — `customer_data._pool_build` passes it into `cust._cust_rev` three times - and into `_cadence_bulk`, so it decides what `rev`/`ly`/`ltm`/`aov`/`status` MEAN. Enforcing - a BU purely as a post-filter would keep the row list right and silently consolidate every - number. So the pushdown survives as a DERIVATION OF the declared filter rather than a second - wall beside it, and `perm_scope.derive_pool_scope` falls back to the legacy `bus`/`agent` - derivation for any record the migration has not reached yet. - """ - import core.perm_scope as perm_scope - return perm_scope.derive_pool_scope(session.user, MODULE) - - -def _pool_stamp(rt, team_id, agent): - """The cached pool's build timestamp — the DATA STAMP in every measure-memo key, so a pool - refresh invalidates the memoised answers exactly when the underlying rows changed.""" - entry = rt.pool_cache.get(("pool", team_id, agent)) - return entry[0] if isinstance(entry, tuple) and entry else 0 - - -def _measure_err(tag, e): - try: - import harness.telemetry as _tel - _tel.error(f"api:{tag}", e) - except Exception: - pass - - -# ══════════════════════════ THE TENANT-WIDE STRATUM, ON THE CUSTOMER TOPIC (W38-T20 / D-425) ══ -# -# ⛔⛔ WHY THIS FILE GREW A SHARED STRATUM AT ALL. `core/shared_overlay.py` has been generic since -# W29-T62, `routes_products.py` has merged it since W30-T36 and `routes_tables.py` since W38-T16 — -# and the CUSTOMER topic had neither a write door nor a read merge. Every user-created column here -# lives in `data[username]`, so two accounts looking at "the same" column are looking at two -# columns. That is fine for a private note and fatal for a ROUTE ORDER: a visit sequence one rep -# can see and their colleague cannot is not a plan, it is a rumour. -# -# ⛔ ONE SPELLING OF THE BUCKET. `modules.customer_data.TABLE_KEY` is the per-user workspace key -# and `shared_overlay.bucket()` derives `__shared` from it. Resolving it here rather than -# writing the string means the write door and the read merge cannot disagree about where the -# values live — which is exactly the failure T16 found on the materialised `ut_*` tables, where -# `patch_shared_cell` wrote into a bucket no reader ever opened. - - -def _shared_key(): - """The store key this topic's per-user AND tenant-wide strata are both named from.""" - import modules.customer_data as cl - return cl.TABLE_KEY - - -def _customer_table(session): - import core.table_store as table_store - return table_store.make(_shared_key(), st=session.runtime) - - -def shared_fields(st=None): - """`{field_key: Field}` — the columns this topic shares tenant-wide. - - Unscoped on purpose, exactly as `shared_overlay.fields` is: a shared column's EXISTENCE is - tenant-wide by definition. WHO MAY SEE IT is a separate question, answered one layer up by - `perm_scope.hidden_keys` (the per-field grant wall T16 landed), and WHOSE ROWS by `cells`. - """ - from core import field_permissions, shared_overlay - try: - field_permissions.migrate_legacy_fields( - _shared_key(), st=st, grant_topic="customer_data", shared_key=_shared_key()) - return shared_overlay.fields(_shared_key(), st=st) - except Exception: # noqa: BLE001 - # Lenient like every other display read: an unreachable store degrades to "nothing is - # shared yet", never to a 500 on a grid that would otherwise render. The WALL does not - # degrade with it — `field_grant_hidden` hides a marked column it cannot resolve. - return {} - - -def shared_cells(pids, st=None): - """`{"": {key: value}}` for the rows named by `pids`, and ONLY those. - - ⛔ `pids` IS THE ROW WALL, PASSED AND NEVER DEFAULTED. `shared_overlay.cells` refuses an - "everything" read by signature for this reason; the set handed in is the one `grid_assembly` - has already narrowed with `apply_row_scope`, so a cell belonging to the other BU has nothing - to attach itself to. - """ - from core import shared_overlay - try: - return shared_overlay.cells(_shared_key(), list(pids or ()), st=st) - except Exception: # noqa: BLE001 - return {} - - -def _merge_shared_fields(fields, defs, session=None): - """`fields` PLUS the tenant-wide columns this topic declares — `routes_tables._ut_shared_fields` - on the customer topic. - - ⚠ MERGED BEFORE THE WALL, NEVER AFTER. `hidden_keys` is a TRANSITIVE closure, so it must run - on the WHOLE contract: a formula over a shared column that reads a hidden one sits outside the - closure's reach otherwise and carries the hidden value out wearing a second name. It is also - the only order in which `field_grant_hidden` can ever see the `granted` marker at all — merge - afterwards and the per-field wall is inert while every test still passes. - ⚠ A key the canonical contract already declares WINS. A shared column is an ADDITION to this - database's contract, never a redefinition of a column it already has. - """ - if not defs: - return fields - have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} - projected = [] - for k, f in defs.items(): - if k in have: - continue - item = dict(f, source="overlay", shared=True) - if session is not None: - from core import shares - role = shares.role_for( - "field", shares.field_oid("customer_data", k), session.uname, - is_admin=session.admin, st=session.runtime) - if role: - item["sharedRole"] = role - projected.append(item) - return list(fields or ()) + projected - - -def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "", - consume_corrections: bool = True): - """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the - events route (2026-07-31 — the standalone measure gap, owner item 1). - - What it adds over the pre-wave hand-rolled `_payload` loop, and why it replaced it: - - * rows go through `aios_grid.rows_from_pool` — the SAME builder the embedded host uses, - so `lat`/`lon` (the Map view's data), `_created` and every derived column ride each row - by construction instead of by a second loop that drifts. The hand loop was written to - mirror the pre-Map contract and silently dropped the coordinates: the shell's Map view - had nothing to plot ("the Map no longer works"). - * `derived` carries the cohort column's cells AND the measure columns' values, resolved - through `core.measure_resolve` (EXIT-5's extraction of the `_cl_measure_*` family). - Without them every measure column the owner built rendered BLANK in the shell. - * `measures` (the offer) and `measure_sets` (condition answers) are computed here so the - events route can finally validate measure fields/conditions instead of refusing them - (an empty `measure_offer` made `clean_measure_field` reject every create over HTTP). - - Memos live on the TENANT RUNTIME (`rt.measure_memo` / `rt.mset_memo`) — bounded by the - module's own clear-past-cap rule, keyed on (stamp, scope, pool identity, question), nothing - user-shaped in them. - """ - import aios_grid - from core import grid_events, measure_resolve - - import core.perm_scope as perm_scope - - rt = session.runtime - team_id, agent = _team_agent(session) - rows_src = _pool_for(rt, team_id, agent) - # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that - # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so - # scoping here means a row this account may not see never enters ANY of them, rather than - # being filtered out of one payload and surviving in another. - # - # Evaluated against the CANONICAL field list, not the per-user assembled one, for two - # reasons: the assembled list is not built yet (it needs `pids`), and a permanent filter may - # only ever name a canonical field anyway — `routes_admin._clean_perms` validates it against - # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer. - # - # ⭐⭐ OWNER I16 — `st=rt` IS WHAT MAKES A WALL ON A USER-GENERATED COLUMN MEAN ANYTHING - # HERE. *"Permission Filters must be able to filter on user-generated Fields too."* The - # sentence above is exactly why it was needed: the canonical list has no `custom_` column in - # it and these rows are PRE-OVERLAY, so such a leaf denied every row while the editor - # reported the rule saved. With the handle, `perm_scope._enrich_for_wall` merges the - # tenant-wide value for the named column onto a COPY of each row and declares it for the - # evaluator. Nothing else about this call changes, and a caller with no handle still gets - # the wall exactly as it was. - # - # ⛔ THE SAME HANDLE GOES TO `allowed_pids` BELOW, AND THE PAIR IS NOT OPTIONAL. That is the - # WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule - # narrow what an account SEES while leaving what it may PATCH untouched. - rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS, - st=rt) - pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) - # ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because - # `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this - # dict on one request (the write ctx's wall, the read merge, and the closure), and letting each - # take its own copy is the shape D-214 spent a whole ticket removing one document over. - _defs = shared_fields(st=rt) - ws = grid_events.table_workspace( - _ctx_for(session, pids, defs=_defs), allowed_pids=pids, - consume_corrections=consume_corrections) - # ⭐⭐ W38-T20 / D-425 — THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE - # ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves - # `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same - # dict to `rows_from_pool`, so merging HERE reaches both without touching either file. - # - # ⚠ SAFE TO MUTATE, and checked rather than assumed (the same check `product_assembly` - # records): `table_workspace` reads through `store.get`, which deep-copies, so `ws` is a - # detached copy and nothing writes it back. A shared value can never leak INTO the per-user - # bucket by way of this merge. - # ⚠ SHARED WINS PER KEY. The whole point of the stratum is that every reader sees the same - # number, so a per-user leftover under the same key is stale by construction. It is also what - # makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not - # promoted. - _shared = shared_cells(pids, st=rt) - if _shared: - _ov = dict(ws.get("overlays") or {}) - for _pid, _cells in _shared.items(): - _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells} - ws["overlays"] = _ov - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key) - # ⭐⭐ W38-T20 — AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this - # position is load-bearing twice, once for the transitive closure and once because it is the - # only order in which the per-field grant marker is ever presented to `hidden_keys`. - fields = _merge_shared_fields(fields, _defs, session=session) - # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides - # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so - # shipping a dependent formula while withholding its input either leaks the input through - # the formula's value or silently computes a wrong one; only removing both is coherent. - # Applied AFTER workspace_wire because custom + measure columns are what it must cover. - # ⭐ W38-T20 — `st=rt` IS NOT TIDINESS. `perm_scope.field_grant_hidden` resolves a marked - # column against `object_shares`, and without a tenant handle it reads the module-default - # bucket: on any tenant but #0 that finds no grant, and no grant on a MARKED column means - # HIDDEN. So an unthreaded `st` UNDER-shares (a grantee cannot see their own column) rather - # than over-shares — visible and reportable, but still wrong, and `visible_fields`' own note - # requires the two calls to agree about it. - hidden = perm_scope.hidden_keys(session.user, MODULE, fields, st=rt) - if hidden: - fields = [f for f in fields if f.get("key") not in hidden] - # The field LIST and the ROW payload are two different wires. Narrowing only the first - # would leave the value sitting in the second, where anything can read it. - rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] - # ⛔⛔ AND THE OVERLAY DICT IS A THIRD WIRE, WHICH IS NEW THIS TICKET AND WAS A HOLE - # BEFORE IT. `rows_from_pool` iterates `fields`, so a narrowed contract already keeps a - # hidden key off a ROW — but `/workspace` serves `ws["overlays"]` RAW, and that dict is - # where both the per-user cells and (as of the merge above) the tenant-wide ones sit. - # One narrowing cannot speak for a wire it never touches; `routes_tables` writes the same - # sentence over its own `shared_cells`, and `routes_odoo_tables` over its `overlays`. - # ⚠ `ws` AND NOT `workspace`: `routes_grid.workspace` copies the dict ACROSS - # (`workspace["overlays"] = g["ws"].get("overlays")`) after this returns, so narrowing the - # source is what reaches the wire. Narrowing the copy would be narrowing a key that gets - # overwritten a moment later. - _ov = ws.get("overlays") or {} - if _ov: - ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden} - for pid, cells in _ov.items()} - - today = time.strftime("%Y-%m-%d") - stamp = _pool_stamp(rt, team_id, agent) - # ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE - # OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their - # condition answers through helpers that short-circuit on an empty `offer`, so emptying the - # offer there stops the whole feature. `core.measure_resolve` does not take an offer at all: - # `condition_sets` and `column_values` both re-derive their work from the caller's OWN saved - # views and field list. So gating only the offer here would take the Metric kind off the - # picker and refuse new creates while an EXISTING Metric column kept computing and an - # EXISTING measure condition kept resolving — a revoked capability still answering, on the - # grain with the most of them. Three calls, one predicate. - may_metrics = perm_scope.may_metrics(session.user, MODULE) - measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else [] - measure_sets = measure_resolve.condition_sets( - [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp, - rt.mset_memo, on_error=_measure_err) if may_metrics else {} - # The derived channel: cohort membership cells + measure column values, ONE dict — the - # same read-only channel the embed host hands to rows_from_pool. - derived = aios_grid.cohort_cells(lists) - # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried - # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id` - # as its join key — so the merge would have lost the one value every Odoo document joins on. - # - # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS - # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is - # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for - # one fact, which is the class of defect item 12 is about. This channel exists for exactly - # this — a value the host knows per render and the pool has no business storing. - for pid in pids: - derived.setdefault(pid, {})["partner_id"] = pid - # ⭐⭐ W38-T19 — SKIPPED ENTIRELY WHEN THE CAPABILITY IS REVOKED, rather than filtered after. - # `column_values` selects its own subjects (`isinstance(f.get('measure'), dict)`) off the - # field list, so there is no argument that could narrow it; not calling it is the narrowing. - # ⚠ THE COLUMN STAYS AND ITS CELLS GO BLANK, which is this channel's OWN documented degrade - # ("blank is could not compute, 0 is a real zero") and is what the product and user-table - # grains already do under an empty offer. Deleting the column instead would be a second, - # louder behaviour for the same fact on one surface out of three, and it would destroy a - # definition the admin can restore with one tick. - for pid, cells in (measure_resolve.column_values( - fields, team_id, pids, today, stamp, rt.measure_memo, - on_error=_measure_err) if may_metrics else {}).items(): - derived.setdefault(pid, {}).update(cells) - - return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, "derived": derived, - "measures": measures, "measure_sets": measure_sets, "today": today, - "team_id": team_id} - - -def _payload(session: Session): - """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py` - referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by - construction); see `grid_assembly` for what that fixed. - - ⭐ `docs` joined the shape in wave 30 (W30-T37 / contract C4). Named here rather than left to - the reader because a docstring that still lists the OLD shape is a stale comment on correct - code — this repo's D-73 — and it is the first thing anyone greps to learn the payload. - - ⚠ `rows_src` is the SHARED cached list — `rows_from_pool` reads it and builds NEW dicts, - never mutating a cached row (the same-scope-second-user leak rule). - """ - import aios_grid - - g = grid_assembly(session) - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) - # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door - # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is - # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. - # All six `onDoc*` handlers in `CustomerGrid.tsx` read `payload?.docs ? … : undefined`, so an - # ABSENT key — not a broken one — is what has been switching the whole feature off. - # - # ⛔ IMPORTED, NEVER RE-SERIALISED. `core.grid_events.docs_for` is the ONE serialiser and - # `routes_tables` (the `ut_*` scope) calls the SAME function with the same argument order. - # A matching pair here is precisely how the wave-29 close-out reintroduced its own defect in - # the opposite direction inside a single commit ([[one-question-two-normalizers]]). - # ⚠ `g["pids"]` is the row set this session is ALREADY scoped to — `docs_for` has no - # "every document in the tenant" mode to reach for, deliberately. - from core import grid_events as _ge - return {"fields": g["fields"], "rows": rows, - # `today` rides the payload because every relative date condition must resolve against - # the TENANT's day, never the browser's — a client that falls back to its own clock - # disagrees with the server for everyone west of it. - "today": g["today"], - "docs": _ge.docs_for(g["pids"], scope_key="customer", uname=session.uname, - admin=session.admin, st=session.runtime), - "pulled_at": time.strftime("%Y-%m-%d %H:%M")} - - -def _ctx_for(session: Session, pids, defs=None): - """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather - than a phantom in-memory workspace an API request cannot persist. - - `defs` (W38-T20) is this topic's tenant-wide column definitions when the caller already holds - them; absent, they are read. One read per assembly rather than one per question asked of it. - """ - from core import grid_events - return grid_events.EventCtx( - uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[], - # C-PERM: the write wall's field half. Computed from the CANONICAL contract PLUS the - # tenant-wide columns, because `fields=[]` here — the closure only needs the schema, not - # this user's column list, and a runtime column is part of that schema now. - hidden_keys=_hidden_for(session, defs=defs), - admin=session.admin, table=_customer_table(session), st=session.runtime, - fallback_ws=None, seen_ids={}) - - - -def _hidden_for(session: Session, defs=None): - """The fields this session's permissions hide — the write wall's half of C-PERM. - - Read paths strip these from both wires so they cannot be SEEN; this is what stops them - being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the - same schema `routes_admin._clean_perms` validates a hiddenFields entry against. - - ⭐⭐ W38-T20 — AND THE TENANT-WIDE COLUMNS ARE PART OF THAT CONTRACT NOW, WHICH IS A WALL AND - NOT A COMPLETENESS TIDY-UP. `patch_customer` builds its ctx with `fields=payload["fields"]`, - which carries the merged shared columns, so `grid_events.handle_one` would happily accept an - `overlay_patch` naming one. The canonical list cannot mention them (they are created at - runtime), so a wall computed from `aios_grid.FIELDS` alone answers "not hidden" for every - grant-governed column and the write door is open to a reader who was never granted it. - ⚠ Widening the field list can only ever ADD to the hidden set, never remove from it: the - closure hides what it is told to hide plus whatever depends on it. - """ - import aios_grid - import core.perm_scope as perm_scope - - if defs is None: - defs = shared_fields(st=session.runtime) - return perm_scope.hidden_keys( - session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs), - st=session.runtime) - -def allowed_pids(session: Session): - """The pids this session may touch — the POOL's own ids, so the write wall and the read scope - can never disagree. - - Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through - the full payload would pay for a workspace read and a row assembly on every write. - - ⛔ THE PERMANENT FILTER APPLIES HERE TOO, AND FORGETTING IT IS A WRITE-WITHOUT-READ HOLE. - `_pool_rows` is built with the DERIVED pushdown, which expresses only what a `(team_id, - agent)` pair can express. Any part of the wall the pushdown cannot carry — `revenue > 1000`, - a nested group, a condition on any other column — leaves the pool WIDER than the filter. Read - paths close that gap with `apply_row_scope`; without the same call here the write wall would - be the wider set, and a restricted user could PATCH a row this API will not show them. - Same function, same order as `grid_assembly`, so the two walls cannot drift. - - ⛔ AND `st` IS PART OF "SAME FUNCTION, SAME ORDER" (owner I16). `grid_assembly` lends the - tenant handle so a wall naming a user-generated column can be answered at all; without the - identical argument here that rule would narrow the READ and not the WRITE — the drift this - docstring already refuses, wearing a new argument's clothes. - """ - import core.perm_scope as perm_scope - import aios_grid - - rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE, - aios_grid.FIELDS, st=session.runtime) - return frozenset(r["pid"] for r in rows if r.get("pid") is not None) - - -@router.get("/customers") -def customers(session: Session = Depends(module_gate(MODULE))): - return _payload(session) - - -def _split_route_updates(session, updates): - """Split a cell PATCH into `(route order updates, everything else)`. - - ⛔ MATCHED ON THE DECLARED `kind`, NEVER ON THE `route_` PREFIX — the rule `route_order_delete` - and the column menu both state. A prefix is a naming convention; the kind is a declaration. - """ - defs = shared_fields(st=session.runtime) or {} - route, rest = {}, {} - for key, value in (updates or {}).items(): - defn = defs.get(key) - if isinstance(defn, dict) and defn.get("kind") == ROUTE_KIND: - route[key] = value - else: - rest[key] = value - return route, rest - - -def _patch_route_ranks(session, ctx, pid, pool, updates): - """Write visit numbers into the SHARED stratum, keeping the column a permutation. - - ⭐⭐ OWNER ITEM 7 (2026-08-23) — THE AUTO-DEDUPE, IN THE OWNER'S OWN WORDS: *"when you edit a - number from say 13 to 14, what ever was record 14 should automatically change to 13 so its - auto dedupe that way."* That is a TRANSPOSITION, and it is the whole rule: the number you - typed goes on your record, and the number you displaced goes to whoever was holding it. Two - rows move, the set of numbers in the column is unchanged, and no other record is renumbered. - - Three cases, and each one is the same rule read honestly: - - · the target number is FREE -> plain assignment. A gap is not a duplicate, and the owner's - rule is about duplicates. - · the target is HELD and this record already had a number -> the two swap. The named case. - · the target is HELD and this record had NO number (it was not on the route) -> there is no - old number to hand over, so the displaced record goes to the end (`max + 1`). Still one - other row moved, still no duplicate, and nothing loses its place on the route silently. - - Blanking a cell takes the record OFF the route and frees its number. It renumbers nothing: - re-solving is what closes the gaps, and doing it here would silently rewrite a day somebody - is driving. - - ⛔⛔ THE WALLS ARE `grid_events`' OWN, CALLED AND NOT COPIED, AND THAT IS THE PRICE OF - STEPPING OUT OF `overlay_patch`. `_may_edit_field_value` is where the shared-field Share role, - the creator rule and the admin rule already meet; the hidden-key wall is applied first for the - reason `overlay_patch` applies it, namely that `hidden_keys` decides what this caller may READ - and a column they cannot read is not one they may write by naming its key. A second spelling - of either here would be a second one to keep in step — which is the whole reason - `patch_customer` routes everything else through `grid_events` rather than writing the store. - - ⚠ UNIQUENESS IS MAINTAINED OVER THE ROWS THIS CALLER CAN SEE, and that is a limit worth - stating rather than hiding. `shared_overlay.cells` takes the scoped pool BY SIGNATURE (there is - deliberately no "every shared cell in the tenant" call), so the holder of a displaced number is - looked for inside this session's book. In practice the whole column lives there anyway -- - `route_order_write` REFUSES a body carrying a pid outside the writer's pool, so every number in - a route order was written by somebody whose book contained all of them. - """ - from core import grid_events, shared_overlay - - defs = shared_fields(st=session.runtime) or {} - hidden = _hidden_for(session) - #: `{key: what the store holds for THIS record now}`, for every key asked about — refused ones - #: included. See the `_stored` note below. - accepted = {} - #: The keys that actually moved. `patch_customer` reports everything else as REFUSED, so this - #: has to be separate from `accepted`: a refused key still reports a value, and it is the - #: value the caller did NOT ask for. - taken, swaps = set(), [] - - def _held(): - """`{pid: rank}` for this column, over the rows this caller can see. - - Re-read per key rather than hoisted, because a multi-key patch writes between iterations - and the second key must see the first one's result. - """ - out = {} - for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): - if not isinstance(cells, dict): - continue - cell = cells.get(key) - if cell in (None, ""): - continue - try: - out[int(raw_pid)] = int(str(cell).strip()) - except (TypeError, ValueError): - continue - return out - - for key, raw in updates.items(): - defn = defs.get(key) or {} - held = _held() - - def _stored(): - """⛔⛔ A REFUSAL REPORTS WHAT THE CELL ACTUALLY HOLDS, AND THAT IS NOT TIDINESS. - - `patchTopicRow` adopts `body.updates` into the browser's optimistic copy and rolls - back only on a non-2xx. A refused key that is simply ABSENT from `updates` is a key - the client never hears about: the response is 200, the rollback never fires, `adopt` - has nothing to write, and the typed text goes on painting a value the store rejected - until something unrelated forces a refetch. On a `select` the option normaliser made - this unreachable; item 7 made the column an `int`, so `14.5` or a stray letter is now - an ordinary typo away. `patchTopicRow`'s own docstring already states the rule — - keep only what the server actually took — and this is that rule with its hole closed. - """ - n = held.get(pid) - return "" if n is None else str(n) - - if key in hidden: - accepted[key] = _stored() - continue - if not grid_events._may_edit_field_value(ctx, key, defn): - accepted[key] = _stored() - continue - # An empty cell is "not on this route". Legal, and the only way to take a stop off the - # day without re-solving the whole column. - if raw is None or (isinstance(raw, str) and not raw.strip()): - shared_overlay.put_rows(_shared_key(), {pid: {key: ""}}, st=session.runtime) - accepted[key] = "" - taken.add(key) - continue - # ⛔ A BOOL IS AN `int` IN PYTHON, so `True` would store as visit number 1. Excluded by - # name here exactly as `route_order_write` excludes it, rather than by hoping. - if isinstance(raw, bool): - accepted[key] = _stored() - continue - try: - want = int(str(raw).strip()) - except (TypeError, ValueError): - accepted[key] = _stored() - continue - if want < 1: - accepted[key] = _stored() - continue - - prior = held.get(pid) - if prior == want: - accepted[key] = str(want) - taken.add(key) - continue - writes = {pid: {key: str(want)}} - holder = next((p for p, n in held.items() if n == want and p != pid), None) - if holder is not None: - moved = prior if prior is not None else max(held.values()) + 1 - writes[holder] = {key: str(moved)} - swaps.append({"pid": holder, "field": key, "value": str(moved)}) - shared_overlay.put_rows(_shared_key(), writes, st=session.runtime) - accepted[key] = str(want) - taken.add(key) - return accepted, swaps, taken - - -@router.patch("/customers/{pid}") -def patch_customer(pid: int, body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): - """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever. - - Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing - the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall - and the truncation rules live, and a second implementation of those would be a second set of - them to keep in step. The response reports what was ACCEPTED, which is not always what was - asked for. - """ - from core import grid_events - - updates = dict(body or {}) - if not updates: - raise err(400, "empty_patch", "no fields to update") - pool = allowed_pids(session) - if pid not in pool: - # 403, not 404: the pid may well exist — it is simply not in this session's book, and - # saying "no such customer" would confirm the opposite to anyone who guessed right. - raise err(403, "out_of_scope", "that customer is not in your book") - payload = _payload(session) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pool, fields=payload["fields"], - admin=session.admin, fallback_ws=None, seen_ids={}, - hidden_keys=_hidden_for(session), table=_customer_table(session), - st=session.runtime) - - # ⭐⭐ OWNER ITEM 7 (2026-08-23) — A ROUTE RANK IS EDITED HERE, NOT THROUGH `overlay_patch`. - # - # ⚠ AND **NOT** BECAUSE THE ORDINARY PATH WRITES THE WRONG STRATUM. It does not: - # `table_store.patch_overlay` has split tenant-wide keys off to `shared_overlay` since W38-T20 - # (D-423), so a route cell already lands where every reader looks. What the ordinary path - # cannot do is the half the owner actually asked for — *"make sure that none of the number can - # be a duplicate ... when you edit a number from say 13 to 14, what ever was record 14 should - # automatically change to 13"*. That is a write to a SECOND record, decided by the first one's - # OLD value, and nothing in `overlay_patch` has a reason to look at either. - # - # ⛔ IT ALSO OWNS THE VALIDATION, because item 7 changed the column's type. As a `select` the - # declared choices refused anything that was not a rank; as an `int` `overlay_patch` stores - # whatever string it is handed, and "third" in a visit-order column is not a slightly wrong - # value — it is a stop with no place in the sequence. - # - # ⛔ INTERCEPTED SERVER-SIDE, NEVER IN THE CLIENT. Every path that can change a cell — typing, - # paste, fill, undo, the record drawer — comes through this one door; a client-side branch - # would have to be repeated at each of them and would be wrong at the first one nobody - # remembered. - route_updates, updates = _split_route_updates(session, updates) - - if updates: - try: - grid_events.handle_one( - {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch", - "pid": pid, "updates": updates}, ctx) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable — your change was not saved") - - route_accepted, route_swaps, route_taken = ({}, [], set()) - if route_updates: - route_accepted, route_swaps, route_taken = _patch_route_ranks( - session, ctx, pid, pool, route_updates) - - # What actually landed, read back from the store rather than echoed from the request: a - # refused key or a truncated value must not be reported as accepted. - stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None) - .get("overlays") or {}).get(str(pid)) or {} - accepted = {k: stored.get(k) for k in updates if k in stored} - refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k])) - # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so - # read-your-writes within this runtime is a property of the design rather than of a - # write-through step somebody has to remember. (It was a write-through step while the whole - # payload was cached on a scope key — the arrangement that leaked one user's notes to - # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module - # docstring says why, and Postgres is the fix. - out = {"ok": True, "pid": pid, "updates": {**accepted, **route_accepted}} - # ⚠ `route_taken` AND NOT `route_accepted`. A refused rank still reports a value — the one - # the store holds — so membership of `updates` no longer means the write landed. - refused = [k for k in refused if k not in route_updates] - refused += [k for k in route_updates if k not in route_taken] - if refused: - out["refused"] = sorted(refused) - # ⭐ THE SWAP CHANGED A ROW THE CLIENT NEVER TYPED IN, so it has to be told. The optimistic - # copy in the browser covers `pid` only; the record that gave up the number it held would go - # on painting the old one until something else forced a refetch. `patchTopicRow` reads this - # key and drops the rows cache. - if route_swaps: - out["routeSwaps"] = route_swaps - return out - - -# ══════════════════════════════════ THE ROUTE-ORDER COLUMN (W38-T20 / ruling R7 / contract C1) ══ -# -# ⛔⛔ WHY THIS IS A DOOR HERE AND NOT A KIND IN THE COLUMN MENU. `ColumnMenu.onCreate` is the only -# persistence the menu has, and it lands a PER-USER `custom_` column through -# `aios_grid._field_extras`, which is a strict allowlist: a route-order bag created that way is -# silently stripped and its values are private to their author. That is the wave-19 `image` -# failure verbatim ("created, named, configured, gone", recorded in `_clean_geocode`'s own -# docstring) plus a done-when clause that cannot hold, because a colleague reading a per-user -# stratum gets a clean 200 with nothing in it. The `geocode` pseudo-kind is the precedent for the -# PICKER; its persistence half does not transfer. -# -# ⭐ SO THE COLUMN IS BORN SHARED. `shared_overlay.put_field` stores the definition verbatim (no -# allowlist), which is what lets the input fingerprint ride the DEFINITION rather than the rows — -# `shared_overlay._value` RAISES on a dict, so `{order, inputsHash}` could never be one cell, and -# one solve fingerprints the whole cohort identically anyway, so per-row would be the same string -# written N times. -# -# ⭐ AND THE NUMBER ON THE RECORD IS THE INVERSE OF THE PLANNER'S ANSWER. `mapProjection.planRoute` -# returns `order`, where `order[i]` is WHICH STOP is visited i-th; the cell holds the RANK. The -# client inverts it with `routeRanks` (gated in `map.test.ts` at the desktop shape, over a fixture -# chosen so the two differ); this door then refuses anything that is not a clean 1..N, so a -# truncated or double-posted body cannot land as a half-route. - -#: The key prefix every route-order column wears. `custom_` and `measure_` are the two existing -#: created-column namespaces and both are PER USER; this one is tenant-wide, so it takes its own -#: rather than borrowing a prefix whose readers assume a per-user home. -ROUTE_KEY_PREFIX = "route_" - -#: The marker on the stored definition that says WHAT this column is. Read by the GET below and by -#: the client; never inferred from the key, because a prefix is a naming convention and a -#: convention is not a declaration. -ROUTE_KIND = "route_order" - -#: ⛔⛔ THE TOPIC A FIELD GRANT IS NAMED UNDER, AND IT IS **NOT** THE STORE BUCKET. Two namespaces -#: meet on this column and they are spelled differently: -#: -#: the STRATUM lives at `shared_overlay.bucket(customer_data.TABLE_KEY)` -#: = `customer_table_workspace__shared` -#: the GRANT is named `shares.field_oid(, key)` -#: = `customer_data:` -#: -#: `perm_scope.hidden_keys(user, module, fields)` hands its `module` argument straight through to -#: `field_grant_hidden`, which builds the oid from it. On a `ut_*` database the module argument IS -#: the table key, so W38-T16 never had to tell them apart; on a REGISTRY topic they differ, and a -#: door that claims the grant under the bucket name writes a record the wall will never look for. -#: MEASURED, not reasoned: the first run of this ticket's gate did exactly that, and user B was -#: refused a column that had been shared with them through the real share door, with a 200 at -#: every step ([[one-question-two-normalizers]]). -SHARE_TOPIC = MODULE - - -def _route_slug(label): - """A stable store key from a human label. Lower case, non-alphanumerics collapsed to `_`.""" - import re as _re - slug = _re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return f"{ROUTE_KEY_PREFIX}{slug[:48]}" if slug else "" - - -def _occupied(defs, base_fields=()): - """The keys and case-folded labels a new route column must not land on.""" - occupied_keys = {str(key) for key in (defs or {})} - occupied_labels = set() - for field in list((defs or {}).values()) + list(base_fields or ()): - if not isinstance(field, dict): - continue - label = " ".join(str(field.get("label") or "").split()).casefold() - if label: - occupied_labels.add(label) - key = str(field.get("key") or "").strip() - if key: - occupied_keys.add(key) - return occupied_keys, occupied_labels - - -def _next_route_label(defs, base_fields=()): - """Return the next human route-field label without colliding with a visible field. - - ⭐ OWNER ITEM 5 (2026-08-23) — THE WORD IS "ROUTE", NOT "DESTINATION". Owner: *"Instead of - calling it 'Destination 1' etc. for when we saved the route to a field, we just call it - 'Route 1' so it's Route 1, Route 2, Route 3."* A saved column holds the ORDER of a whole day, - so "Destination 1" read as the first stop rather than as the first route, which is exactly - backwards from what the numbers in it mean. - - ⛔ NO MIGRATION, AND THAT IS DELIBERATE. Columns already minted as `Destination N` keep their - label: rewriting a tenant-wide column name that other people's saved views point at is not a - rename this door was asked for. The owner renames one through `route_order_rename` below, per - owner item 6 of the same instruction. - - ⚠ THE KEY IS STILL DERIVED FROM THE LABEL, so the default now slugs to `route_route_1`. The - redundant segment is NOT tidied away: `ROUTE_KEY_PREFIX` is what keeps a route column out of - the namespace this database owns, and stripping it here would make the labels "Route 1" and - "1" collide on one key. The key is never on screen. - """ - occupied_keys, occupied_labels = _occupied(defs, base_fields) - n = 1 - while True: - label = f"Route {n}" - if label.casefold() not in occupied_labels and _route_slug(label) not in occupied_keys: - return label - n += 1 - - -def _clean_route_depot(raw): - """The optional origin stored once on a route-order definition, never on a customer cell.""" - if raw is None: - return None - if not isinstance(raw, dict): - raise err(400, "bad_depot", "a depot is an address with latitude and longitude, or null") - address = " ".join(str(raw.get("address") or "").split())[:200] - lat, lon = raw.get("lat"), raw.get("lon") - if (not address or isinstance(lat, bool) or isinstance(lon, bool) or - not isinstance(lat, (int, float)) or not isinstance(lon, (int, float))): - raise err(400, "bad_depot", "a depot needs an address and numeric latitude and longitude") - lat, lon = float(lat), float(lon) - if not (math.isfinite(lat) and math.isfinite(lon) and abs(lat) <= 90 and abs(lon) <= 180): - raise err(400, "bad_depot", "the depot latitude or longitude is outside the map") - return {"address": address, "lat": lat, "lon": lon} - - -def _route_defs(session: Session): - """This topic's route-order columns MINUS the ones this session was not granted. - - ⛔ THE WALL IS THE SAME ONE THE GRID USES, NOT A SECOND OPINION. `hidden_keys` is where T16 - put the per-field grant check, so filtering on it here means the listing, the grid contract - and the row payload agree by construction. A column a reader cannot see must not appear here - either: the done-when says *does not see the field at all*, and a picker that names a column - whose values are withheld has already leaked its existence. - """ - import aios_grid - import core.perm_scope as perm_scope - - all_defs = shared_fields(st=session.runtime) or {} - defs = {k: v for k, v in all_defs.items() - if isinstance(v, dict) and v.get("kind") == ROUTE_KIND} - if not defs: - return {} - hide = perm_scope.hidden_keys( - session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs), - st=session.runtime) - return {k: v for k, v in defs.items() if k not in hide} - - -def _own_route_fork(session: Session, key: str): - """This caller's PRIVATE copy of a route column, if a per-user write ever forked one. - - ⛔⛔ THE FORK IS REAL, AND IT IS WHAT BROKE DELETE. Traced through `grid_events.field_upsert` - rather than assumed: `shared_field` is `bool(shared_prior.get('custom'))` and a route - definition carries no `custom` key, so the shared branch does not take it; the key is neither - `custom_` nor `measure_`; it IS in the merged contract, so the `key in field_by_key` branch - does — and that branch's own body is what writes `note`, `format` and `agg`, landing them in - THIS USER'S field definitions through `TableStore.save_field`. `_merge_shared_fields` then - SKIPS the tenant-wide definition, because a key the contract already declares wins, and from - that moment the person is reading a private copy of a shared column. - - ⚠ WHICH DOORS STILL FORK, AS OF 2026-08-23, because "the fork is fixed" would be too broad. - The Description and the Name are CLOSED: owner item 6 routed `onNote` and `onRename` to - `route_order_rename` below. The Edit-field pane's **Format** row (`onFormat`, unconditional) - and its **Summary** row (`onAggregate`, whose `isUserTable` arm is false on this registry - topic) both still reach `saveField`, so either one still mints a fork. They are left open on - purpose: closing them needs this door to accept `format` and `agg`, which is a wider change - than the delete the owner reported. This function is what keeps that recoverable rather than - permanent. - - ⛔ WHY THAT MADE THE DELETE 404 RATHER THAN MERELY MISBEHAVE. The first delete found the - tenant-wide definition, dropped it and answered 200 — and the column came straight back on the - next read, because the fork was still declaring it. The second attempt found nothing in - `shared_fields` and answered *"that column is not a route order column on this database"* - about a column sitting on screen. Owner, 2026-08-23: *"I can't even delete the Field route - now?"* Both halves are answered here: a fork is FOUND, and `route_order_delete` drops it WITH - the definition rather than leaving it to redeclare the column. - - ⚠ `kind` IS THE TEST, NEVER MERE PRESENCE. `TableStore.workspace` merges the tenant-wide - column summary over this stratum and will mint a bare `{'agg': ...}` entry for a key the user - has never touched (W36-T25, whose own comment says it must be able to CREATE an entry). - Reading presence would call that stub a fork and scrub a column nobody had forked. - """ - try: - ws = _customer_table(session).workspace(session.uname, consume_corrections=False) - except Exception: # noqa: BLE001 - return None - entry = (ws.get("fields") or {}).get(key) - if not isinstance(entry, dict) or entry.get("kind") != ROUTE_KIND: - return None - return entry - - -@router.get("/customers/route-order") -def route_order_list(session: Session = Depends(module_gate(MODULE))): - """The route-order columns this session may see, with the fingerprint each was solved from. - - ⭐⭐ THE FINGERPRINT IS WHY THIS ROUTE EXISTS RATHER THAN THE CLIENT READING `fields`. - Staleness is DERIVED, never stored: what is written down is the INPUT FINGERPRINT the numbers - were produced from, so *is this order still current* is a question asked at READ time against - what is on screen NOW, and can never itself be out of date. A stored `stale: true` is a fact - about a moment that has already passed. - """ - out = [] - for key, defn in sorted(_route_defs(session).items()): - route = defn.get("route") if isinstance(defn.get("route"), dict) else {} - out.append({ - "key": key, - "label": defn.get("label") or key, - "inputsHash": str(route.get("inputsHash") or ""), - "roundTrip": bool(route.get("roundTrip")), - "startPid": route.get("startPid"), - "stops": route.get("stops"), - "depot": route.get("depot") if isinstance(route.get("depot"), dict) else None, - "solvedAt": route.get("solvedAt") or "", - "solvedBy": defn.get("createdBy") or "", - # Who may re-solve it. The same creator-or-admin wall the write door enforces, said on - # the way out so the client can grey the control instead of discovering a 403. - "mine": bool(session.admin - or str(defn.get("createdBy") or "") == session.uname), - }) - # ⭐⭐ OWNER ITEM 5 — THE DEFAULT NAME IS ALLOCATED **HERE**, NEVER ON THE CLIENT. - # - # The panel prompts for a route name and pre-fills it. Computing that pre-fill from the - # `fields` list above would compute it from the columns this session may SEE: `_route_defs` - # drops every route column the per-field wall hides, so a user with no grant on - # `route_route_1` would be offered "Route 1" as their default, send it, and be refused - # `field_key_taken` on a name the app itself put in the box. Allocated over the WHOLE shared - # stratum, the suggestion can never name a column that already exists. - # - # ⚠ IT LEAKS NOTHING. What travels is the first FREE name, which is a fact about absence. - import aios_grid - return {"fields": out, - "nextLabel": _next_route_label(shared_fields(st=session.runtime) or {}, - aios_grid.FIELDS)} - - -@router.post("/customers/route-order") -def route_order_write(body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): - """Create (or re-solve) a tenant-wide route-order column and fill it, in ONE call. - - `{label?, field?, ranks: {"": }, inputsHash, roundTrip?, startPid?, depot?}` - - When `label` is omitted for a new column, the door allocates the next unused `Destination N` - label. This keeps route creation one-click while preserving unique, readable field names. - - ⛔ THE DEFINITION IS WRITTEN FIRST AND THE GRANT CLAIMED SECOND, which is `patch_shared_cell`'s - order and it is deliberate: the window where a column is MARKED and UNCLAIMED fails CLOSED - (governed, nobody granted, so only an admin and the creator see it, and an admin can share - it). The other order would leave a grant record pointing at nothing. - - ⛔ RE-SOLVING IS CREATOR-OR-ADMIN, WHICH CREATING IS NOT. Writing these numbers changes what - every account in the workspace reads, at once, for the whole cohort. That is the wall - `routes_tables.delete_shared_field` already applies to the destructive half of this stratum, - and a new door does not get to inherit the loose half of an asymmetry somebody has flagged. - A permitted teammate READS the numbers; they do not silently re-plan somebody's day. - """ - from core import shared_overlay - import core.perm_scope as perm_scope - from routes_grid import MAX_BULK_ROWS - - body = body if isinstance(body, dict) else {} - label = " ".join(str(body.get("label") or "").split())[:120] - import aios_grid - defs = shared_fields(st=session.runtime) or {} - requested_key = str(body.get("field") or "").strip() - if requested_key: - key = requested_key - elif label: - key = _route_slug(label) - else: - label = _next_route_label(defs, aios_grid.FIELDS) - key = _route_slug(label) - if not key: - raise err(400, "bad_request", "a name is required for the route order column") - if not key.startswith(ROUTE_KEY_PREFIX): - raise err(400, "bad_field_key", - f"a route order column's key starts with '{ROUTE_KEY_PREFIX}', so it cannot " - f"collide with a column this database already owns") - if key in {f.get("key") for f in aios_grid.FIELDS}: - raise err(400, "field_key_taken", - "this database already has a column with that key") - - existing = defs.get(key) if isinstance(defs.get(key), dict) else None - if existing is not None: - # ⛔⛔ A REFUSAL MUST NOT DESCRIBE A COLUMN THE CALLER CANNOT SEE. The two refusals below - # name the column's KIND and its CREATOR, which is exactly the information the field wall - # exists to withhold: a stranger who guesses the key would otherwise learn that a route - # order exists on this database and who planned it. `routes_shares._can_see_object` makes - # the same choice for the same reason (a non-grantee gets the answer a non-existent id - # gets). The name is already taken either way, so the honest refusal says only that. - if key in perm_scope.hidden_keys( - session.user, MODULE, - _merge_shared_fields(list(aios_grid.FIELDS), defs), st=session.runtime): - raise err(400, "field_key_taken", - "that column name is already in use on this database") - if existing.get("kind") != ROUTE_KIND: - raise err(400, "not_a_route_column", - "that column is shared but it is not a route order column, so re-solving " - "it would overwrite values this door did not write") - owner = str(existing.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a route order can be re-solved by the person who planned it or by an " - f"administrator. This one was planned by {owner or 'somebody else'}, and " - f"re-solving it would change the visit numbers for every account at once") - - if "depot" in body: - depot = _clean_route_depot(body.get("depot")) - else: - prior_route = (existing or {}).get("route") if isinstance(existing, dict) else {} - depot = prior_route.get("depot") if isinstance(prior_route, dict) else None - - raw = body.get("ranks") - if not isinstance(raw, dict) or not raw: - raise err(400, "bad_ranks", 'expected {ranks: {"": }}') - # ⛔ REPORTED, NEVER TRUNCATED (standing rule 1's second sentence, and `MAX_BULK_ROWS`' own - # note). The ceiling is `routes_grid`'s so there is ONE of them, not two that drift. - if len(raw) > MAX_BULK_ROWS: - raise err(400, "too_many_rows", - f"at most {MAX_BULK_ROWS} records per route; this one carried {len(raw)}") - - pool = allowed_pids(session) - ranks, not_in_pool, bad_value = {}, [], [] - for raw_pid, value in raw.items(): - try: - pid = int(raw_pid) - except (TypeError, ValueError): - not_in_pool.append(str(raw_pid)[:40]) - continue - # ⚠ THE POOL IS THE WALL, and it is the SAME predicate the read path applies - # (`apply_row_scope` inside `allowed_pids`), so a caller cannot number a record they - # could not be shown, including one in the other business unit. - if pid not in pool: - not_in_pool.append(str(raw_pid)[:40]) - continue - # ⛔ A BOOL IS AN `int` IN PYTHON and `True` would store as a visit number 1. Excluded by - # name rather than by hoping nobody sends one. - if isinstance(value, bool) or not isinstance(value, int) or value < 1: - bad_value.append(str(raw_pid)[:40]) - continue - ranks[pid] = value - - # ⛔⛔ A PARTLY HONOURED ROUTE IS NOT A ROUTE, AND THIS REFUSES THE WHOLE CALL RATHER THAN - # WRITING THE PART IT COULD. Caught by this ticket's own gate: a body carrying one record - # outside the caller's book still produced a clean 1..N over what was left, so the door minted - # a permanent tenant-wide column and filled it with a SHORTER route than the one that was - # solved. Every number in it was plausible and the day was wrong. - # ⚠ REPORTED, WITH THE SAMPLE, which is standing rule 1's second sentence: the refusal names - # what it could not take, so the caller can fix it rather than guess. - if not_in_pool: - raise err(400, "rows_not_in_your_book", - f"{len(not_in_pool)} of those records are not in your book, so the route " - f"cannot be written as it was solved. First few: " - f"{', '.join(sorted(not_in_pool)[:5])}") - if bad_value: - raise err(400, "rows_not_a_visit_number", - f"a visit number is a whole number from 1 upwards; {len(bad_value)} records " - f"carried something else. First few: {', '.join(sorted(bad_value)[:5])}") - if not ranks: - raise err(400, "no_rows_in_your_book", - "none of those records are in your book, so there is nothing to number") - # ⛔ A ROUTE IS A SEQUENCE, SO THE NUMBERS ARE 1..N WITH NO REPEAT AND NO GAP. A body that - # arrives truncated, doubled or partly applied would otherwise land as a plausible half - # route: every row carrying a number, and the day in the wrong order. - seq = sorted(ranks.values()) - if seq != list(range(1, len(seq) + 1)): - raise err(400, "not_a_sequence", - f"a route order is the numbers 1 to {len(seq)}, each used once. This one " - f"carried {len(seq)} records numbered up to {seq[-1]} with " - f"{len(seq) - len(set(seq))} repeated") - - stamp = time.strftime("%Y-%m-%d %H:%M") - defn = { - "key": key, - "label": label or (existing or {}).get("label") or key, - # ⭐⭐ OWNER ITEM 7 (2026-08-23) — THE SAVED ROUTE IS A **NUMBER** COLUMN. - # - # Owner: *"the Route field once saved should be saved as a number Field. Make sure we can - # edit it, BUT also make sure that none of the number can be a duplicate, so its like a - # number order field."* It used to be a `select` whose declared choices were the strings - # "1".."N", which sorted and rendered as a list rather than as an order and forced a - # re-solve to widen the vocabulary before a stop could be numbered past N. - # - # ⛔ UNIQUENESS IS NOT A PROPERTY OF THE TYPE, SO IT LIVES AT THE WRITE DOOR. `int` has no - # "no duplicates" flag anywhere in this contract; `_patch_route_ranks` in `patch_customer` - # is what keeps the column a permutation, by SWAPPING rather than by refusing. - # - # ⚠ EXISTING COLUMNS ARE NOT MIGRATED, they are UPGRADED BY RE-SOLVE. This dict is - # rebuilt whole on every write, so the first re-solve of a `Destination N` column turns it - # into a number column; one nobody re-solves keeps working as the select it was, and the - # write door reads the KIND rather than the type, so both edit identically. - "type": "int", - "source": "overlay", - "shared": True, - "kind": ROUTE_KIND, - # ⭐⭐ OWNER, 2026-08-23 — A NEW ROUTE READS AS **PRIVATE**, NEVER "Shared with everyone". - # - # Owner: *"Right now the Route is 'Shared with everyone' in terms of the Field status when - # i check the Route Field. It should always default to private first."* - # - # ⛔ A CLASSIFICATION FIX, NOT A TIGHTENING, AND THE DIFFERENCE IS THE WHOLE POINT. This - # column was ALREADY private in the only sense that governs a reader: `FIELD_GRANT_MARK` - # plus the empty-entry grant claimed below means creator-and-admin and nobody else. What - # was wrong was the WORD ON SCREEN. `FieldsHidePanel` sections the field list by - # `types.fieldEditMode`, which is `cleanFieldPermissions(field.permissions, - # "collaborative")` — so a definition carrying NO permissions bag fell to that fallback - # and was filed under "Shared with everyone" while being shared with nobody at all. - # - # ⚠ AND IT MOVES NO WALL, CHECKED RATHER THAN REASONED. `grid_events._may_edit_field_value` - # short-circuits on `definition.get('shared') or definition.get('granted')` BEFORE it reads - # `stored_permissions`, so the creator's own rank edits and `_patch_route_ranks`' swap are - # decided by `_field_share_role` and not by this bag; the client twin `mayEditField` takes - # the same branch in the same order. `field_permissions.migrate_legacy_fields` only ever - # rewrites a PER-USER field carrying `custom: True`, which a route definition is not, so it - # leaves this key alone. `fieldEditMode` has exactly one other consumer: none. - # - # ⛔ NO MIGRATION, AND IT IS THE SAME DELIBERATE CHOICE `_next_route_label` MAKES ABOUT - # `Destination N`. This dict is rebuilt whole on every write, so a column minted before - # today gains the bag on its next RE-SOLVE and not one moment sooner — until then it - # keeps reading "Shared with everyone" in the Hide fields panel while being shared with - # nobody. Backfilling every stored route definition on a read is a write nobody asked - # for, on a tenant-wide stratum, triggered by opening a page; re-solving is one click and - # it is the click the owner is already making. Stated here rather than left to be - # rediscovered as "the fix did not work". - "permissions": {"edit": "personal"}, - # ⭐⭐ THE PER-FIELD GRANT MARKER (T16). It is an EXPLICIT write-once declaration and it is - # what makes the wall fail CLOSED: a grant wall has the opposite absence-polarity to a - # deny wall, so keying visibility on "does a grant record exist" would publish this column - # to the whole tenant on one unreadable read of `object_shares`. Stamped, never inferred. - perm_scope.FIELD_GRANT_MARK: True, - "createdBy": (existing or {}).get("createdBy") or session.uname, - # ⭐ THE FINGERPRINT RIDES THE DEFINITION, ONCE. One `planRoute` run solves the whole - # cohort, so this string is identical for every record in it; per row it would be the same - # value written N times, and `shared_overlay._value` raises on a dict anyway. - "route": { - "inputsHash": str(body.get("inputsHash") or "")[:64], - "roundTrip": bool(body.get("roundTrip")), - "startPid": int(body["startPid"]) if isinstance(body.get("startPid"), int) - and not isinstance(body.get("startPid"), bool) else None, - "stops": len(ranks), - "depot": depot, - "solvedAt": stamp, - }, - } - shared_overlay.put_field(_shared_key(), key, defn, st=session.runtime) - if existing is None: - try: - import core.shares as shares - # ⚠ THE EMPTY ENTRY LIST IS THE POINT. `set_grants` keeps a record with an owner and - # no entries, so "shared with nobody" is STORED and is a different fact from "never - # shared". Without the owner the column is unmanageable: `may_administer` fails closed - # on an ownerless record, so nobody could ever share it. - shares.set_grants("field", shares.field_oid(SHARE_TOPIC, key), [], - owner=session.uname, st=session.runtime) - except Exception: # noqa: BLE001 - # The mark is already written, so a failed claim fails CLOSED: governed, nobody - # granted, admin-and-creator only. Recoverable. The other order is not. - pass - - # ⛔ AND THE RECORDS THAT LOST THEIR NUMBER ARE CLEARED. A re-solve over a SMALLER cohort - # would otherwise leave the previous run's ranks sitting on the records that dropped out — - # plausible integers, from a route nobody is driving. `""` is the house spelling of an empty - # overlay cell (`rows_from_pool` defaults an absent one to exactly that), so this needs no new - # vocabulary and no tombstone nobody else reads. - stale = {} - for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): - if not isinstance(cells, dict) or cells.get(key) in (None, ""): - continue - try: - gone = int(raw_pid) - except (TypeError, ValueError): - continue - if gone not in ranks: - stale[gone] = {key: ""} - written = shared_overlay.put_rows( - _shared_key(), {**{p: {key: str(v)} for p, v in ranks.items()}, **stale}, - st=session.runtime) - - out = {"ok": True, "field": key, "label": defn["label"], "type": "int", - "stops": len(ranks), "cleared": len(stale), - "rows_written": len(written), "inputsHash": defn["route"]["inputsHash"], - "depot": defn["route"]["depot"], - "solvedAt": stamp} - return out - - -@router.delete("/customers/route-order/{field_key}") -def route_order_delete(field_key: str, - session: Session = Depends(module_gate(MODULE))): - """Remove a Destination column and every visit number in it. - - ⛔⛔ WHY THIS DOOR HAD TO EXIST. `route_order_write` mints a TENANT-WIDE column and nothing - could ever remove it: the grid's own Delete is offered for a per-user `custom_` column - (`menuField.custom && !menuField.shared`) or for a `ut_*` definition field, and a route order - is neither — it is a SHARED column on a registry topic. So `Destination 1` was permanent for - the whole workspace, which is [[reachable-is-not-the-same-as-built]] from the other end: - `shared_overlay.drop_field` was complete and correct and no route on this topic called it. - - ⛔ CREATOR-OR-ADMIN, the same wall `routes_tables.delete_shared_field` applies and for the - same reason: writing a cell changes a value, dropping the column deletes that value for every - account at once. It is deliberately NOT the looser wall `route_order_write` uses for CREATE. - - ⛔ AND A CALLER WHO CANNOT SEE THE COLUMN GETS THE ANSWER A NONEXISTENT KEY GETS. The refusals - below would otherwise teach a stranger that a route order exists on this database and who - planned it, which is exactly what the per-field wall withholds — the choice already argued at - `route_order_write`'s `field_key_taken` and in `routes_shares._can_see_object`. - """ - from core import shared_overlay - import core.perm_scope as perm_scope - import core.table_store as table_store - import aios_grid - - key = str(field_key or "").strip() - all_defs = shared_fields(st=session.runtime) or {} - defn = all_defs.get(key) if isinstance(all_defs.get(key), dict) else None - # ⭐⭐ OWNER, 2026-08-23 — A PRIVATE FORK IS ALSO A COLUMN TO DELETE. See `_own_route_fork` - # above: once any per-user write has forked this key, the fork is what the person is looking - # at and it OUTLIVES a drop of the tenant-wide definition. Answering 404 about a column that - # is on screen is exactly the refusal the owner hit. - fork = _own_route_fork(session, key) - subject = defn or fork - unknown = err(404, "unknown_field", - "that column is not a route order column on this database") - if not subject: - raise unknown - # The wall FIRST, so a hidden column is indistinguishable from an absent one. - # ⚠ ASKED ABOUT THE TENANT-WIDE DEFINITION ONLY, and that is not a hole. The wall reads - # `FIELD_GRANT_MARK` off the MERGED contract; with the definition already gone there is no - # marked column left for it to hide, and a fork lives in this caller's OWN stratum, which no - # grant has ever governed. Not asking when there is nothing to ask about beats asking of a - # contract that no longer carries the key and reading the empty answer as "not hidden". - if defn and key in perm_scope.hidden_keys( - session.user, MODULE, - _merge_shared_fields(list(aios_grid.FIELDS), all_defs), st=session.runtime): - raise unknown - # ⛔ KIND-CHECKED, NOT PREFIX-CHECKED. `ROUTE_KEY_PREFIX` is a naming convention and a - # convention is not a declaration (the comment on `ROUTE_KIND` says so); a door that deleted - # by prefix would happily drop a shared column somebody else's feature owns. - if subject.get("kind") != ROUTE_KIND: - raise err(400, "not_a_route_column", - "that column is shared but it is not a route order column, so this door will " - "not remove it") - owner = str(subject.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a route order column can be removed by the person who planned it or by an " - f"administrator. This one was planned by {owner or 'somebody else'}, and " - f"removing it would delete the visit numbers for every account at once") - - dropped = bool(defn) and shared_overlay.drop_field(_shared_key(), key, st=session.runtime) - # ⛔⛔ AND THE FORK GOES IN THE SAME CALL. Dropping only the tenant-wide definition is what - # made the first delete look like it had worked and then undo itself: the fork still declares - # the column, `_merge_shared_fields` still yields to it, and the next paint brings it back. - # `delete_field` scrubs this user's overlay values under the key too, which is right: the - # tenant-wide cells went with `drop_field` above, and a private leftover would resurface under - # whatever column later took the key. - if fork is not None: - try: - table_store.make(_shared_key(), st=session.runtime).delete_field(session.uname, key) - except Exception: # noqa: BLE001 - # The definition is already gone, so a failed fork scrub must not turn a delete that - # succeeded into a 500. Worst case the column lingers for this one account until its - # next write, which is recoverable; a 500 over completed work is not. - pass - # ⭐⭐ THE GRANT DIES WITH THE COLUMN. `route_order_write` claims - # `shares.field_oid(SHARE_TOPIC, key)` on create, so skipping this would leave a grant record - # pointing at nothing — a ghost in every receiver's "Shared with me" that 404s on open, and - # worse, one that silently re-arms on the next column to take the key, because `drop_field` - # scrubs the cells precisely so the key CAN be re-used. - # ⚠ Same order and same tolerance as `routes_tables.delete_shared_field`: the column is - # already gone, so a failed release must not turn a completed delete into a 500. - try: - import core.shares as shares - shares.drop_objects([("field", shares.field_oid(SHARE_TOPIC, key))], st=session.runtime) - except Exception: # noqa: BLE001 - pass - return {"ok": True, "field": key, "dropped": bool(dropped or fork is not None)} - - -@router.patch("/customers/route-order/{field_key}") -def route_order_rename(field_key: str, body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): - """Rename a route order column, or write its description. `{label?, note?}`. - - ⭐⭐ OWNER ITEM 6 (2026-08-23) — "Edit field" MUST BE ABLE TO RENAME THIS COLUMN. - - A route order is a SHARED column on the customers REGISTRY topic. The grid's Edit-field pane - offers a Name box only when the host supplies `onRename`, and the host supplies it only for - `isUserSchemaField` — a per-user `custom_` column, or a `ut_*` definition field. A route order - is neither, so the pane showed a DISABLED Name box reading *"A source field keeps its name and - type from the data source"* about a column the person had created themselves an hour earlier. - That is the same shape as owner item 3's missing Delete, one stratum over. - - ⛔⛔ THE KEY IS FROZEN. `_route_slug` derives the store key from the label AT CREATION, and - only there. Re-slugging on a rename would leave every written visit number sitting under the - old key in `shared_overlay`, every saved view's `colId` pointing at a column that no longer - exists, and every field grant naming an oid nobody can reach — a rename that looks perfect on - a fresh column and quietly empties a used one. The label moves; nothing else does. - - ⛔ CREATOR-OR-ADMIN, and a caller who cannot SEE the column gets the answer a nonexistent key - gets. Both walls are `route_order_delete`'s, verbatim and for its reasons: this changes what - every account in the workspace reads, and a refusal that said "forbidden" would confirm to a - stranger that a route order exists here and who planned it. - - ⛔⛔ THE DESCRIPTION RIDES THIS DOOR TOO, AND IT HAD TO. The Edit-field pane writes a - description through `onNote`, which is `saveField` — a PER-USER `field_upsert`. Traced through - `grid_events`: a route key is not `custom`-marked in the shared stratum, so the `shared_field` - branch does not take it; it IS in the merged contract, so the `key in field_by_key` branch - does, and it stores `{**base, note}` in THIS USER'S field definitions. `_merge_shared_fields` - then skips the shared definition, because a key the contract already declares wins — so that - user is left reading a private copy of a tenant-wide column, frozen at the label it had when - they typed the description, and a later rename through this door is invisible to them. - Pre-existing, and unreachable enough to have gone unnoticed; owner item 6 makes that pane the - place people go for route columns, so it stops being unreachable on the same day. - """ - from core import shared_overlay - import core.perm_scope as perm_scope - import aios_grid - - body = body if isinstance(body, dict) else {} - key = str(field_key or "").strip() - label = " ".join(str(body.get("label") or "").split())[:120] - # ⚠ `"note" in body` AND NOT a truthiness test: an empty string is how a description is - # CLEARED, and a door that read it as "unchanged" would leave one nobody can delete. - note = str(body.get("note") or "")[:2000] if "note" in body else None - if not label and note is None: - raise err(400, "bad_request", "a route order column needs a name") - if "label" in body and not label: - raise err(400, "bad_request", "a route order column needs a name") - - all_defs = shared_fields(st=session.runtime) or {} - defn = all_defs.get(key) if isinstance(all_defs.get(key), dict) else None - unknown = err(404, "unknown_field", - "that column is not a route order column on this database") - if not defn: - raise unknown - if key in perm_scope.hidden_keys( - session.user, MODULE, - _merge_shared_fields(list(aios_grid.FIELDS), all_defs), st=session.runtime): - raise unknown - if defn.get("kind") != ROUTE_KIND: - raise err(400, "not_a_route_column", - "that column is shared but it is not a route order column, so this door will " - "not rename it") - owner = str(defn.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a route order column can be renamed by the person who planned it or by an " - f"administrator. This one was planned by {owner or 'somebody else'}, and its " - f"name is what every account in the workspace reads") - - # ⚠ THE COLLISION CHECK LOOKS AT LABELS AND NOT AT KEYS, because the key is frozen: this - # rename can never take another column's key, only its NAME. Two columns wearing one name on - # the same grid is the confusion `_next_route_label` exists to prevent at creation, so the - # rename door refuses it too. The column's OWN current label is excluded, so re-saving an - # unchanged name is a no-op rather than a refusal. - _keys, occupied_labels = _occupied( - {k: v for k, v in all_defs.items() if k != key}, aios_grid.FIELDS) - if label and label.casefold() in occupied_labels: - raise err(400, "field_label_taken", - "this database already has a column with that name") - - patch = dict(defn) - if label: - patch["label"] = label - if note is not None: - patch["note"] = note - shared_overlay.put_field(_shared_key(), key, patch, st=session.runtime) - return {"ok": True, "field": key, "label": patch.get("label") or key, - "note": patch.get("note") or ""} +"""routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a). + +Two things happen here that did not happen in the pre-wave `main.py`: + + 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole + book, to anyone holding the shared APP_PASSWORD. Now the pool is built with + `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a + Fisch row and an agent-linked login never receives another rep's book. The scope is applied + at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist + in this response. + + 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home + for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and + whichever process you asked last was right. Reads and writes now both go through + `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3). + +⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is +cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API +container until its cache is refreshed, and vice versa. Deleting the fork removes the second +SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4, +owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient +proof is single-process and would report green on exactly the thing that is still broken. +""" +import math +import time + +from fastapi import APIRouter, Body, Depends + +import scope_cache +from deps import Session, err, module_gate, perms + +router = APIRouter(prefix="/api/v1") + +#: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a +#: 403 rather than an empty table that looks like "you have no customers". +MODULE = "customer_data" + +_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before + + +def _pool_rows(session: Session): + """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that + may be shared between users. + + ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is + `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is + genuinely scope-shaped — two users with the same BU and the same book are asking the same + question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation). + + The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped + and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_` + and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table + workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e. + PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one + would have been served the FIRST one's private notes and private columns. The scope key was + right for the pool and wrong for everything wrapped around it. + + It survived a green 129-check battery because every fixture user had a DISTINCT + `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not + express the bug. `verify_api.py` now carries a same-scope second user for exactly this. + """ + rt = session.runtime + team_id, agent = _team_agent(session) + return _pool_for(rt, team_id, agent) + + +def _pool_for(rt, team_id, agent): + """The cached pool for an explicit scope — session-free so the prewarm thread and the + stale-refresh path can call it. STALE-WHILE-REFRESH (scope_cache): once a copy exists no + request blocks on the 10–30s Odoo rebuild again; only a scope's FIRST-ever build does. + + DEBT-2 (2026-08-04): while the tenant's RESOLVED Odoo source is PAUSED this path never + goes live — it serves the in-process copy at any age, else the persisted pause-time + snapshot (restart-safe), else answers 503. Never a WIDER scope's snapshot: handing a + scoped user the consolidated rows would widen their book, which is worse than an error.""" + import modules.customer_data as cl + import routes_keychain + + key = ("pool", team_id, agent) + + if routes_keychain.odoo_paused(rt): + hit = rt.pool_cache.get(key) + if hit: + return hit[1] + snap = routes_keychain.load_pool_snapshot(rt, team_id, agent) + if snap is not None: + rt.pool_cache[key] = snap # seed, so the memo stamps stay coherent + return snap[1] + raise err(503, "connector_paused", + "this data source is paused and no snapshot exists for your scope — " + "an admin can resume it under Settings → Connectors") + + def _build(): + # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled + # builder the Streamlit page uses — passing the scope here is what makes the isolation a + # property of the QUERY instead of a filter someone can forget to apply downstream. + return cl.pool(agent, team_id) + + def _evict(): + # Bounded: a scope cache that only ever grows is a memory leak in a shared process. + if len(rt.pool_cache) > 16: + for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]: + rt.pool_cache.pop(stale, None) + + return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, _build, _evict) + + +def warm_default(rt): + """Boot prewarm: the consolidated pool `(None, None)` — the scope every admin and every + all-BU account lands on. Called from main.py's prewarm thread only.""" + _pool_for(rt, None, None) + + +def _team_agent(session: Session): + """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by + `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree. + + ⛔ WAVE 15 R1: THIS NOW COMES FROM THE PERMANENT FILTER (C-PERM amendment 3). `team_id` is + not a row filter — `customer_data._pool_build` passes it into `cust._cust_rev` three times + and into `_cadence_bulk`, so it decides what `rev`/`ly`/`ltm`/`aov`/`status` MEAN. Enforcing + a BU purely as a post-filter would keep the row list right and silently consolidate every + number. So the pushdown survives as a DERIVATION OF the declared filter rather than a second + wall beside it, and `perm_scope.derive_pool_scope` falls back to the legacy `bus`/`agent` + derivation for any record the migration has not reached yet. + """ + import core.perm_scope as perm_scope + return perm_scope.derive_pool_scope(session.user, MODULE) + + +def _pool_stamp(rt, team_id, agent): + """The cached pool's build timestamp — the DATA STAMP in every measure-memo key, so a pool + refresh invalidates the memoised answers exactly when the underlying rows changed.""" + entry = rt.pool_cache.get(("pool", team_id, agent)) + return entry[0] if isinstance(entry, tuple) and entry else 0 + + +def _measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: + pass + + +# ══════════════════════════ THE TENANT-WIDE STRATUM, ON THE CUSTOMER TOPIC (W38-T20 / D-425) ══ +# +# ⛔⛔ WHY THIS FILE GREW A SHARED STRATUM AT ALL. `core/shared_overlay.py` has been generic since +# W29-T62, `routes_products.py` has merged it since W30-T36 and `routes_tables.py` since W38-T16 — +# and the CUSTOMER topic had neither a write door nor a read merge. Every user-created column here +# lives in `data[username]`, so two accounts looking at "the same" column are looking at two +# columns. That is fine for a private note and fatal for a ROUTE ORDER: a visit sequence one rep +# can see and their colleague cannot is not a plan, it is a rumour. +# +# ⛔ ONE SPELLING OF THE BUCKET. `modules.customer_data.TABLE_KEY` is the per-user workspace key +# and `shared_overlay.bucket()` derives `__shared` from it. Resolving it here rather than +# writing the string means the write door and the read merge cannot disagree about where the +# values live — which is exactly the failure T16 found on the materialised `ut_*` tables, where +# `patch_shared_cell` wrote into a bucket no reader ever opened. + + +def _shared_key(): + """The store key this topic's per-user AND tenant-wide strata are both named from.""" + import modules.customer_data as cl + return cl.TABLE_KEY + + +def _customer_table(session): + import core.table_store as table_store + return table_store.make(_shared_key(), st=session.runtime) + + +def shared_fields(st=None): + """`{field_key: Field}` — the columns this topic shares tenant-wide. + + Unscoped on purpose, exactly as `shared_overlay.fields` is: a shared column's EXISTENCE is + tenant-wide by definition. WHO MAY SEE IT is a separate question, answered one layer up by + `perm_scope.hidden_keys` (the per-field grant wall T16 landed), and WHOSE ROWS by `cells`. + """ + from core import field_permissions, shared_overlay + try: + field_permissions.migrate_legacy_fields( + _shared_key(), st=st, grant_topic="customer_data", shared_key=_shared_key()) + return shared_overlay.fields(_shared_key(), st=st) + except Exception: # noqa: BLE001 + # Lenient like every other display read: an unreachable store degrades to "nothing is + # shared yet", never to a 500 on a grid that would otherwise render. The WALL does not + # degrade with it — `field_grant_hidden` hides a marked column it cannot resolve. + return {} + + +def shared_cells(pids, st=None): + """`{"": {key: value}}` for the rows named by `pids`, and ONLY those. + + ⛔ `pids` IS THE ROW WALL, PASSED AND NEVER DEFAULTED. `shared_overlay.cells` refuses an + "everything" read by signature for this reason; the set handed in is the one `grid_assembly` + has already narrowed with `apply_row_scope`, so a cell belonging to the other BU has nothing + to attach itself to. + """ + from core import shared_overlay + try: + return shared_overlay.cells(_shared_key(), list(pids or ()), st=st) + except Exception: # noqa: BLE001 + return {} + + +def _merge_shared_fields(fields, defs, session=None): + """`fields` PLUS the tenant-wide columns this topic declares — `routes_tables._ut_shared_fields` + on the customer topic. + + ⚠ MERGED BEFORE THE WALL, NEVER AFTER. `hidden_keys` is a TRANSITIVE closure, so it must run + on the WHOLE contract: a formula over a shared column that reads a hidden one sits outside the + closure's reach otherwise and carries the hidden value out wearing a second name. It is also + the only order in which `field_grant_hidden` can ever see the `granted` marker at all — merge + afterwards and the per-field wall is inert while every test still passes. + ⚠ A key the canonical contract already declares WINS. A shared column is an ADDITION to this + database's contract, never a redefinition of a column it already has. + """ + if not defs: + return fields + have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} + projected = [] + for k, f in defs.items(): + if k in have: + continue + item = dict(f, source="overlay", shared=True) + if session is not None: + from core import shares + role = shares.role_for( + "field", shares.field_oid("customer_data", k), session.uname, + is_admin=session.admin, st=session.runtime) + if role: + item["sharedRole"] = role + projected.append(item) + return list(fields or ()) + projected + + +def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "", + consume_corrections: bool = True): + """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the + events route (2026-07-31 — the standalone measure gap, owner item 1). + + What it adds over the pre-wave hand-rolled `_payload` loop, and why it replaced it: + + * rows go through `aios_grid.rows_from_pool` — the SAME builder the embedded host uses, + so `lat`/`lon` (the Map view's data), `_created` and every derived column ride each row + by construction instead of by a second loop that drifts. The hand loop was written to + mirror the pre-Map contract and silently dropped the coordinates: the shell's Map view + had nothing to plot ("the Map no longer works"). + * `derived` carries the cohort column's cells AND the measure columns' values, resolved + through `core.measure_resolve` (EXIT-5's extraction of the `_cl_measure_*` family). + Without them every measure column the owner built rendered BLANK in the shell. + * `measures` (the offer) and `measure_sets` (condition answers) are computed here so the + events route can finally validate measure fields/conditions instead of refusing them + (an empty `measure_offer` made `clean_measure_field` reject every create over HTTP). + + Memos live on the TENANT RUNTIME (`rt.measure_memo` / `rt.mset_memo`) — bounded by the + module's own clear-past-cap rule, keyed on (stamp, scope, pool identity, question), nothing + user-shaped in them. + """ + import aios_grid + from core import grid_events, measure_resolve + + import core.perm_scope as perm_scope + + rt = session.runtime + team_id, agent = _team_agent(session) + rows_src = _pool_for(rt, team_id, agent) + # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that + # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so + # scoping here means a row this account may not see never enters ANY of them, rather than + # being filtered out of one payload and surviving in another. + # + # Evaluated against the CANONICAL field list, not the per-user assembled one, for two + # reasons: the assembled list is not built yet (it needs `pids`), and a permanent filter may + # only ever name a canonical field anyway — `routes_admin._clean_perms` validates it against + # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer. + # + # ⭐⭐ OWNER I16 — `st=rt` IS WHAT MAKES A WALL ON A USER-GENERATED COLUMN MEAN ANYTHING + # HERE. *"Permission Filters must be able to filter on user-generated Fields too."* The + # sentence above is exactly why it was needed: the canonical list has no `custom_` column in + # it and these rows are PRE-OVERLAY, so such a leaf denied every row while the editor + # reported the rule saved. With the handle, `perm_scope._enrich_for_wall` merges the + # tenant-wide value for the named column onto a COPY of each row and declares it for the + # evaluator. Nothing else about this call changes, and a caller with no handle still gets + # the wall exactly as it was. + # + # ⛔ THE SAME HANDLE GOES TO `allowed_pids` BELOW, AND THE PAIR IS NOT OPTIONAL. That is the + # WRITE wall to this one's READ wall; lending it here alone would make a user-generated rule + # narrow what an account SEES while leaving what it may PATCH untouched. + rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS, + st=rt) + pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) + # ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because + # `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this + # dict on one request (the write ctx's wall, the read merge, and the closure), and letting each + # take its own copy is the shape D-214 spent a whole ticket removing one document over. + _defs = shared_fields(st=rt) + ws = grid_events.table_workspace( + _ctx_for(session, pids, defs=_defs), allowed_pids=pids, + consume_corrections=consume_corrections) + # ⭐⭐ W38-T20 / D-425 — THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE + # ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves + # `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same + # dict to `rows_from_pool`, so merging HERE reaches both without touching either file. + # + # ⚠ SAFE TO MUTATE, and checked rather than assumed (the same check `product_assembly` + # records): `table_workspace` reads through `store.get`, which deep-copies, so `ws` is a + # detached copy and nothing writes it back. A shared value can never leak INTO the per-user + # bucket by way of this merge. + # ⚠ SHARED WINS PER KEY. The whole point of the stratum is that every reader sees the same + # number, so a per-user leftover under the same key is stale by construction. It is also what + # makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not + # promoted. + _shared = shared_cells(pids, st=rt) + if _shared: + _ov = dict(ws.get("overlays") or {}) + for _pid, _cells in _shared.items(): + _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells} + ws["overlays"] = _ov + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key) + # ⭐⭐ W38-T20 — AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this + # position is load-bearing twice, once for the transitive closure and once because it is the + # only order in which the per-field grant marker is ever presented to `hidden_keys`. + fields = _merge_shared_fields(fields, _defs, session=session) + # ⭐⭐ W41-T01 / RULING R5 / CONTRACTS C1 + C8 — **THE THREE BADGES, STAMPED ONCE, HERE.** + # + # C1 makes `field_permissions.field_class` the ONE producer of `{origin, audience, sharedBy, + # values, owner}`; C8 fixes the wire key as `class`. This is the only line on the customer + # topic that calls it. + # + # ⛔ IN `grid_assembly` AND NOT IN `_merge_shared_fields`, WHICH IS THE OTHER CANDIDATE AND IS + # THE WRONG ONE FOR TWO SEPARATE REASONS. First, that helper cannot badge EVERY column: it + # returns `fields` untouched when `defs` is empty and otherwise only builds the PROJECTED + # tenant-wide additions, so the canonical contract and every `custom_`/`measure_` column would + # leave it unbadged — and the done-when is every field. Second, it is called from five places + # (`_hidden_for`, `_route_defs`, and the route write/delete/rename doors) that want a field + # list purely to ask the WALL a question and throw it away; badging there is a registry read + # per column on paths that never reach a client. `grid_assembly` is the single assembly BOTH + # doors serve verbatim — `_payload` returns `g["fields"]` on `/customers` and `routes_grid` + # assigns `workspace["fields"] = g["fields"]` on `/workspace` — so stamping here is the only + # position in which the two wires cannot disagree about a column's badges. + # + # ⚠ AFTER THE MERGE AND BEFORE THE WALL, and the position is load-bearing in both directions. + # After, or the tenant-wide columns (Supplier's twins, every route column) are not in the list + # to badge. Before, so `hidden_keys` below deletes the badge along with the column it belongs + # to: a reader who may not see a column never receives a `class` bag describing who owns it or + # who it is shared with. Stamping after the wall would leave the badge on nothing; stamping + # somewhere later would put it past the narrowing entirely. + # + # ⚠ `values_shared=set(_defs)` REUSES THE DICT READ ONCE AT THE TOP OF THIS ASSEMBLY, so the + # third badge costs no store read at all, and `field_classes` opens `object_shares` ONCE for + # the whole list rather than once per column (see its docstring — that is the D-214 shape). + # ⚠ THE TWO KEYS ARE DIFFERENT STRINGS: `_shared_key()` names the BUCKET, `"customer_data"` + # names the registry TOPIC — the same pair `shared_fields` and `_merge_shared_fields` already + # pass, and swapping them finds no grant for any column. + # + # ⛔ A NEW DICT PER COLUMN, NEVER AN IN-PLACE STAMP. These entries are per-request copies today + # (`aios_grid.fields_from_workspace` does `field = dict(base)`, `_merge_shared_fields` does + # `dict(f, ...)`) — but `class` carries THIS viewer's `sharedBy` and `owner`, so an aliased + # definition would be one account's answer served to the next reader of the same cached + # structure. The copy makes that impossible to reintroduce rather than merely untrue now, which + # is the same rule `_payload` states over `rows_src`. + # ⛔ AND ONLY `class`. C8's `usage`, `agg` and `descriptionEdited` belong to other tickets, and + # C8 is explicit that an absent key means "not built yet", never "false". + try: + from core import field_permissions as _fp + _classes = _fp.field_classes(fields, session.uname, table_key=_shared_key(), + grant_topic=SHARE_TOPIC, values_shared=set(_defs), st=rt) + fields = [dict(f, **{"class": _classes[f["key"]]}) + if isinstance(f, dict) and _classes.get(f.get("key")) else f + for f in fields] + # ⭐⭐ C8's `descriptionEdited`, ON THE SAME PASS AND FOR A MEASURED REASON. W41-T11 shipped + # the producer (`user_tables.description_edited`, R19's custody mark scoped to the + # description) and stopped at its own one-file fence; an audit across the lane branches + # afterwards found `filter-kit/fieldClass.ts::descriptionEditedOf` ALREADY READING THIS KEY + # with nothing sending it. A consumer waiting on a producer is the same dead feature as the + # reverse, and both halves were green. + # ⚠ IT RIDES THE `class` TRY DELIBERATELY. Both are display reads on the same list at the + # same position, so one lenient block is one failure mode instead of two, and a tenant whose + # registry will not open loses both badges together rather than half a row. + from core import user_tables as _ut_desc + fields = [dict(f, descriptionEdited=_ut_desc.description_edited(f)) + if isinstance(f, dict) else f + for f in fields] + except Exception: # noqa: BLE001 + # Lenient like every other display read on this assembly. C8's absent-key polarity is what + # makes this degrade safe: a client renders NOTHING for a missing `class` rather than a + # wrong badge, so a registry that will not open costs the badges and not the grid. + pass + # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides + # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so + # shipping a dependent formula while withholding its input either leaks the input through + # the formula's value or silently computes a wrong one; only removing both is coherent. + # Applied AFTER workspace_wire because custom + measure columns are what it must cover. + # ⭐ W38-T20 — `st=rt` IS NOT TIDINESS. `perm_scope.field_grant_hidden` resolves a marked + # column against `object_shares`, and without a tenant handle it reads the module-default + # bucket: on any tenant but #0 that finds no grant, and no grant on a MARKED column means + # HIDDEN. So an unthreaded `st` UNDER-shares (a grantee cannot see their own column) rather + # than over-shares — visible and reportable, but still wrong, and `visible_fields`' own note + # requires the two calls to agree about it. + hidden = perm_scope.hidden_keys(session.user, MODULE, fields, st=rt) + if hidden: + fields = [f for f in fields if f.get("key") not in hidden] + # The field LIST and the ROW payload are two different wires. Narrowing only the first + # would leave the value sitting in the second, where anything can read it. + rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] + # ⛔⛔ AND THE OVERLAY DICT IS A THIRD WIRE, WHICH IS NEW THIS TICKET AND WAS A HOLE + # BEFORE IT. `rows_from_pool` iterates `fields`, so a narrowed contract already keeps a + # hidden key off a ROW — but `/workspace` serves `ws["overlays"]` RAW, and that dict is + # where both the per-user cells and (as of the merge above) the tenant-wide ones sit. + # One narrowing cannot speak for a wire it never touches; `routes_tables` writes the same + # sentence over its own `shared_cells`, and `routes_odoo_tables` over its `overlays`. + # ⚠ `ws` AND NOT `workspace`: `routes_grid.workspace` copies the dict ACROSS + # (`workspace["overlays"] = g["ws"].get("overlays")`) after this returns, so narrowing the + # source is what reaches the wire. Narrowing the copy would be narrowing a key that gets + # overwritten a moment later. + _ov = ws.get("overlays") or {} + if _ov: + ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden} + for pid, cells in _ov.items()} + + today = time.strftime("%Y-%m-%d") + stamp = _pool_stamp(rt, team_id, agent) + # ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE + # OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their + # condition answers through helpers that short-circuit on an empty `offer`, so emptying the + # offer there stops the whole feature. `core.measure_resolve` does not take an offer at all: + # `condition_sets` and `column_values` both re-derive their work from the caller's OWN saved + # views and field list. So gating only the offer here would take the Metric kind off the + # picker and refuse new creates while an EXISTING Metric column kept computing and an + # EXISTING measure condition kept resolving — a revoked capability still answering, on the + # grain with the most of them. Three calls, one predicate. + may_metrics = perm_scope.may_metrics(session.user, MODULE) + measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else [] + measure_sets = measure_resolve.condition_sets( + [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp, + rt.mset_memo, on_error=_measure_err) if may_metrics else {} + # The derived channel: cohort membership cells + measure column values, ONE dict — the + # same read-only channel the embed host hands to rows_from_pool. + derived = aios_grid.cohort_cells(lists) + # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried + # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id` + # as its join key — so the merge would have lost the one value every Odoo document joins on. + # + # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS + # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is + # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for + # one fact, which is the class of defect item 12 is about. This channel exists for exactly + # this — a value the host knows per render and the pool has no business storing. + for pid in pids: + derived.setdefault(pid, {})["partner_id"] = pid + # ⭐⭐ W38-T19 — SKIPPED ENTIRELY WHEN THE CAPABILITY IS REVOKED, rather than filtered after. + # `column_values` selects its own subjects (`isinstance(f.get('measure'), dict)`) off the + # field list, so there is no argument that could narrow it; not calling it is the narrowing. + # ⚠ THE COLUMN STAYS AND ITS CELLS GO BLANK, which is this channel's OWN documented degrade + # ("blank is could not compute, 0 is a real zero") and is what the product and user-table + # grains already do under an empty offer. Deleting the column instead would be a second, + # louder behaviour for the same fact on one surface out of three, and it would destroy a + # definition the admin can restore with one tick. + for pid, cells in (measure_resolve.column_values( + fields, team_id, pids, today, stamp, rt.measure_memo, + on_error=_measure_err) if may_metrics else {}).items(): + derived.setdefault(pid, {}).update(cells) + + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "derived": derived, + "measures": measures, "measure_sets": measure_sets, "today": today, + "team_id": team_id} + + +def _payload(session: Session): + """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py` + referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by + construction); see `grid_assembly` for what that fixed. + + ⭐ `docs` joined the shape in wave 30 (W30-T37 / contract C4). Named here rather than left to + the reader because a docstring that still lists the OLD shape is a stale comment on correct + code — this repo's D-73 — and it is the first thing anyone greps to learn the payload. + + ⚠ `rows_src` is the SHARED cached list — `rows_from_pool` reads it and builds NEW dicts, + never mutating a cached row (the same-scope-second-user leak rule). + """ + import aios_grid + + g = grid_assembly(session) + rows = aios_grid.rows_from_pool( + g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) + # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door + # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is + # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. + # All six `onDoc*` handlers in `CustomerGrid.tsx` read `payload?.docs ? … : undefined`, so an + # ABSENT key — not a broken one — is what has been switching the whole feature off. + # + # ⛔ IMPORTED, NEVER RE-SERIALISED. `core.grid_events.docs_for` is the ONE serialiser and + # `routes_tables` (the `ut_*` scope) calls the SAME function with the same argument order. + # A matching pair here is precisely how the wave-29 close-out reintroduced its own defect in + # the opposite direction inside a single commit ([[one-question-two-normalizers]]). + # ⚠ `g["pids"]` is the row set this session is ALREADY scoped to — `docs_for` has no + # "every document in the tenant" mode to reach for, deliberately. + from core import grid_events as _ge + return {"fields": g["fields"], "rows": rows, + # `today` rides the payload because every relative date condition must resolve against + # the TENANT's day, never the browser's — a client that falls back to its own clock + # disagrees with the server for everyone west of it. + "today": g["today"], + "docs": _ge.docs_for(g["pids"], scope_key="customer", uname=session.uname, + admin=session.admin, st=session.runtime), + "pulled_at": time.strftime("%Y-%m-%d %H:%M")} + + +def _ctx_for(session: Session, pids, defs=None): + """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather + than a phantom in-memory workspace an API request cannot persist. + + `defs` (W38-T20) is this topic's tenant-wide column definitions when the caller already holds + them; absent, they are read. One read per assembly rather than one per question asked of it. + """ + from core import grid_events + return grid_events.EventCtx( + uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[], + # C-PERM: the write wall's field half. Computed from the CANONICAL contract PLUS the + # tenant-wide columns, because `fields=[]` here — the closure only needs the schema, not + # this user's column list, and a runtime column is part of that schema now. + hidden_keys=_hidden_for(session, defs=defs), + admin=session.admin, table=_customer_table(session), st=session.runtime, + fallback_ws=None, seen_ids={}) + + + +def _hidden_for(session: Session, defs=None): + """The fields this session's permissions hide — the write wall's half of C-PERM. + + Read paths strip these from both wires so they cannot be SEEN; this is what stops them + being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the + same schema `routes_admin._clean_perms` validates a hiddenFields entry against. + + ⭐⭐ W38-T20 — AND THE TENANT-WIDE COLUMNS ARE PART OF THAT CONTRACT NOW, WHICH IS A WALL AND + NOT A COMPLETENESS TIDY-UP. `patch_customer` builds its ctx with `fields=payload["fields"]`, + which carries the merged shared columns, so `grid_events.handle_one` would happily accept an + `overlay_patch` naming one. The canonical list cannot mention them (they are created at + runtime), so a wall computed from `aios_grid.FIELDS` alone answers "not hidden" for every + grant-governed column and the write door is open to a reader who was never granted it. + ⚠ Widening the field list can only ever ADD to the hidden set, never remove from it: the + closure hides what it is told to hide plus whatever depends on it. + """ + import aios_grid + import core.perm_scope as perm_scope + + if defs is None: + defs = shared_fields(st=session.runtime) + return perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs), + st=session.runtime) + +def allowed_pids(session: Session): + """The pids this session may touch — the POOL's own ids, so the write wall and the read scope + can never disagree. + + Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through + the full payload would pay for a workspace read and a row assembly on every write. + + ⛔ THE PERMANENT FILTER APPLIES HERE TOO, AND FORGETTING IT IS A WRITE-WITHOUT-READ HOLE. + `_pool_rows` is built with the DERIVED pushdown, which expresses only what a `(team_id, + agent)` pair can express. Any part of the wall the pushdown cannot carry — `revenue > 1000`, + a nested group, a condition on any other column — leaves the pool WIDER than the filter. Read + paths close that gap with `apply_row_scope`; without the same call here the write wall would + be the wider set, and a restricted user could PATCH a row this API will not show them. + Same function, same order as `grid_assembly`, so the two walls cannot drift. + + ⛔ AND `st` IS PART OF "SAME FUNCTION, SAME ORDER" (owner I16). `grid_assembly` lends the + tenant handle so a wall naming a user-generated column can be answered at all; without the + identical argument here that rule would narrow the READ and not the WRITE — the drift this + docstring already refuses, wearing a new argument's clothes. + """ + import core.perm_scope as perm_scope + import aios_grid + + rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE, + aios_grid.FIELDS, st=session.runtime) + return frozenset(r["pid"] for r in rows if r.get("pid") is not None) + + +@router.get("/customers") +def customers(session: Session = Depends(module_gate(MODULE))): + return _payload(session) + + +def _split_route_updates(session, updates): + """Split a cell PATCH into `(route order updates, everything else)`. + + ⛔ MATCHED ON THE DECLARED `kind`, NEVER ON THE `route_` PREFIX — the rule `route_order_delete` + and the column menu both state. A prefix is a naming convention; the kind is a declaration. + """ + defs = shared_fields(st=session.runtime) or {} + route, rest = {}, {} + for key, value in (updates or {}).items(): + defn = defs.get(key) + if isinstance(defn, dict) and defn.get("kind") == ROUTE_KIND: + route[key] = value + else: + rest[key] = value + return route, rest + + +def _patch_route_ranks(session, ctx, pid, pool, updates): + """Write visit numbers into the SHARED stratum, keeping the column a permutation. + + ⭐⭐ OWNER ITEM 7 (2026-08-23) — THE AUTO-DEDUPE, IN THE OWNER'S OWN WORDS: *"when you edit a + number from say 13 to 14, what ever was record 14 should automatically change to 13 so its + auto dedupe that way."* That is a TRANSPOSITION, and it is the whole rule: the number you + typed goes on your record, and the number you displaced goes to whoever was holding it. Two + rows move, the set of numbers in the column is unchanged, and no other record is renumbered. + + Three cases, and each one is the same rule read honestly: + + · the target number is FREE -> plain assignment. A gap is not a duplicate, and the owner's + rule is about duplicates. + · the target is HELD and this record already had a number -> the two swap. The named case. + · the target is HELD and this record had NO number (it was not on the route) -> there is no + old number to hand over, so the displaced record goes to the end (`max + 1`). Still one + other row moved, still no duplicate, and nothing loses its place on the route silently. + + Blanking a cell takes the record OFF the route and frees its number. It renumbers nothing: + re-solving is what closes the gaps, and doing it here would silently rewrite a day somebody + is driving. + + ⛔⛔ THE WALLS ARE `grid_events`' OWN, CALLED AND NOT COPIED, AND THAT IS THE PRICE OF + STEPPING OUT OF `overlay_patch`. `_may_edit_field_value` is where the shared-field Share role, + the creator rule and the admin rule already meet; the hidden-key wall is applied first for the + reason `overlay_patch` applies it, namely that `hidden_keys` decides what this caller may READ + and a column they cannot read is not one they may write by naming its key. A second spelling + of either here would be a second one to keep in step — which is the whole reason + `patch_customer` routes everything else through `grid_events` rather than writing the store. + + ⚠ UNIQUENESS IS MAINTAINED OVER THE ROWS THIS CALLER CAN SEE, and that is a limit worth + stating rather than hiding. `shared_overlay.cells` takes the scoped pool BY SIGNATURE (there is + deliberately no "every shared cell in the tenant" call), so the holder of a displaced number is + looked for inside this session's book. In practice the whole column lives there anyway -- + `route_order_write` REFUSES a body carrying a pid outside the writer's pool, so every number in + a route order was written by somebody whose book contained all of them. + """ + from core import grid_events, shared_overlay + + defs = shared_fields(st=session.runtime) or {} + hidden = _hidden_for(session) + #: `{key: what the store holds for THIS record now}`, for every key asked about — refused ones + #: included. See the `_stored` note below. + accepted = {} + #: The keys that actually moved. `patch_customer` reports everything else as REFUSED, so this + #: has to be separate from `accepted`: a refused key still reports a value, and it is the + #: value the caller did NOT ask for. + taken, swaps = set(), [] + + def _held(): + """`{pid: rank}` for this column, over the rows this caller can see. + + Re-read per key rather than hoisted, because a multi-key patch writes between iterations + and the second key must see the first one's result. + """ + out = {} + for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): + if not isinstance(cells, dict): + continue + cell = cells.get(key) + if cell in (None, ""): + continue + try: + out[int(raw_pid)] = int(str(cell).strip()) + except (TypeError, ValueError): + continue + return out + + for key, raw in updates.items(): + defn = defs.get(key) or {} + held = _held() + + def _stored(): + """⛔⛔ A REFUSAL REPORTS WHAT THE CELL ACTUALLY HOLDS, AND THAT IS NOT TIDINESS. + + `patchTopicRow` adopts `body.updates` into the browser's optimistic copy and rolls + back only on a non-2xx. A refused key that is simply ABSENT from `updates` is a key + the client never hears about: the response is 200, the rollback never fires, `adopt` + has nothing to write, and the typed text goes on painting a value the store rejected + until something unrelated forces a refetch. On a `select` the option normaliser made + this unreachable; item 7 made the column an `int`, so `14.5` or a stray letter is now + an ordinary typo away. `patchTopicRow`'s own docstring already states the rule — + keep only what the server actually took — and this is that rule with its hole closed. + """ + n = held.get(pid) + return "" if n is None else str(n) + + if key in hidden: + accepted[key] = _stored() + continue + if not grid_events._may_edit_field_value(ctx, key, defn): + accepted[key] = _stored() + continue + # An empty cell is "not on this route". Legal, and the only way to take a stop off the + # day without re-solving the whole column. + if raw is None or (isinstance(raw, str) and not raw.strip()): + shared_overlay.put_rows(_shared_key(), {pid: {key: ""}}, st=session.runtime) + accepted[key] = "" + taken.add(key) + continue + # ⛔ A BOOL IS AN `int` IN PYTHON, so `True` would store as visit number 1. Excluded by + # name here exactly as `route_order_write` excludes it, rather than by hoping. + if isinstance(raw, bool): + accepted[key] = _stored() + continue + try: + want = int(str(raw).strip()) + except (TypeError, ValueError): + accepted[key] = _stored() + continue + if want < 1: + accepted[key] = _stored() + continue + + prior = held.get(pid) + if prior == want: + accepted[key] = str(want) + taken.add(key) + continue + writes = {pid: {key: str(want)}} + holder = next((p for p, n in held.items() if n == want and p != pid), None) + if holder is not None: + moved = prior if prior is not None else max(held.values()) + 1 + writes[holder] = {key: str(moved)} + swaps.append({"pid": holder, "field": key, "value": str(moved)}) + shared_overlay.put_rows(_shared_key(), writes, st=session.runtime) + accepted[key] = str(want) + taken.add(key) + return accepted, swaps, taken + + +@router.patch("/customers/{pid}") +def patch_customer(pid: int, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever. + + Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing + the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall + and the truncation rules live, and a second implementation of those would be a second set of + them to keep in step. The response reports what was ACCEPTED, which is not always what was + asked for. + """ + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + pool = allowed_pids(session) + if pid not in pool: + # 403, not 404: the pid may well exist — it is simply not in this session's book, and + # saying "no such customer" would confirm the opposite to anyone who guessed right. + raise err(403, "out_of_scope", "that customer is not in your book") + payload = _payload(session) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pool, fields=payload["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, + hidden_keys=_hidden_for(session), table=_customer_table(session), + st=session.runtime) + + # ⭐⭐ OWNER ITEM 7 (2026-08-23) — A ROUTE RANK IS EDITED HERE, NOT THROUGH `overlay_patch`. + # + # ⚠ AND **NOT** BECAUSE THE ORDINARY PATH WRITES THE WRONG STRATUM. It does not: + # `table_store.patch_overlay` has split tenant-wide keys off to `shared_overlay` since W38-T20 + # (D-423), so a route cell already lands where every reader looks. What the ordinary path + # cannot do is the half the owner actually asked for — *"make sure that none of the number can + # be a duplicate ... when you edit a number from say 13 to 14, what ever was record 14 should + # automatically change to 13"*. That is a write to a SECOND record, decided by the first one's + # OLD value, and nothing in `overlay_patch` has a reason to look at either. + # + # ⛔ IT ALSO OWNS THE VALIDATION, because item 7 changed the column's type. As a `select` the + # declared choices refused anything that was not a rank; as an `int` `overlay_patch` stores + # whatever string it is handed, and "third" in a visit-order column is not a slightly wrong + # value — it is a stop with no place in the sequence. + # + # ⛔ INTERCEPTED SERVER-SIDE, NEVER IN THE CLIENT. Every path that can change a cell — typing, + # paste, fill, undo, the record drawer — comes through this one door; a client-side branch + # would have to be repeated at each of them and would be wrong at the first one nobody + # remembered. + route_updates, updates = _split_route_updates(session, updates) + + if updates: + try: + grid_events.handle_one( + {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — your change was not saved") + + route_accepted, route_swaps, route_taken = ({}, [], set()) + if route_updates: + route_accepted, route_swaps, route_taken = _patch_route_ranks( + session, ctx, pid, pool, route_updates) + + # What actually landed, read back from the store rather than echoed from the request: a + # refused key or a truncated value must not be reported as accepted. + stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None) + .get("overlays") or {}).get(str(pid)) or {} + accepted = {k: stored.get(k) for k in updates if k in stored} + refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k])) + # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so + # read-your-writes within this runtime is a property of the design rather than of a + # write-through step somebody has to remember. (It was a write-through step while the whole + # payload was cached on a scope key — the arrangement that leaked one user's notes to + # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module + # docstring says why, and Postgres is the fix. + out = {"ok": True, "pid": pid, "updates": {**accepted, **route_accepted}} + # ⚠ `route_taken` AND NOT `route_accepted`. A refused rank still reports a value — the one + # the store holds — so membership of `updates` no longer means the write landed. + refused = [k for k in refused if k not in route_updates] + refused += [k for k in route_updates if k not in route_taken] + if refused: + out["refused"] = sorted(refused) + # ⭐ THE SWAP CHANGED A ROW THE CLIENT NEVER TYPED IN, so it has to be told. The optimistic + # copy in the browser covers `pid` only; the record that gave up the number it held would go + # on painting the old one until something else forced a refetch. `patchTopicRow` reads this + # key and drops the rows cache. + if route_swaps: + out["routeSwaps"] = route_swaps + return out + + +# ══════════════════════════════════ THE ROUTE-ORDER COLUMN (W38-T20 / ruling R7 / contract C1) ══ +# +# ⛔⛔ WHY THIS IS A DOOR HERE AND NOT A KIND IN THE COLUMN MENU. `ColumnMenu.onCreate` is the only +# persistence the menu has, and it lands a PER-USER `custom_` column through +# `aios_grid._field_extras`, which is a strict allowlist: a route-order bag created that way is +# silently stripped and its values are private to their author. That is the wave-19 `image` +# failure verbatim ("created, named, configured, gone", recorded in `_clean_geocode`'s own +# docstring) plus a done-when clause that cannot hold, because a colleague reading a per-user +# stratum gets a clean 200 with nothing in it. The `geocode` pseudo-kind is the precedent for the +# PICKER; its persistence half does not transfer. +# +# ⭐ SO THE COLUMN IS BORN SHARED. `shared_overlay.put_field` stores the definition verbatim (no +# allowlist), which is what lets the input fingerprint ride the DEFINITION rather than the rows — +# `shared_overlay._value` RAISES on a dict, so `{order, inputsHash}` could never be one cell, and +# one solve fingerprints the whole cohort identically anyway, so per-row would be the same string +# written N times. +# +# ⭐ AND THE NUMBER ON THE RECORD IS THE INVERSE OF THE PLANNER'S ANSWER. `mapProjection.planRoute` +# returns `order`, where `order[i]` is WHICH STOP is visited i-th; the cell holds the RANK. The +# client inverts it with `routeRanks` (gated in `map.test.ts` at the desktop shape, over a fixture +# chosen so the two differ); this door then refuses anything that is not a clean 1..N, so a +# truncated or double-posted body cannot land as a half-route. + +#: The key prefix every route-order column wears. `custom_` and `measure_` are the two existing +#: created-column namespaces and both are PER USER; this one is tenant-wide, so it takes its own +#: rather than borrowing a prefix whose readers assume a per-user home. +ROUTE_KEY_PREFIX = "route_" + +#: The marker on the stored definition that says WHAT this column is. Read by the GET below and by +#: the client; never inferred from the key, because a prefix is a naming convention and a +#: convention is not a declaration. +ROUTE_KIND = "route_order" + +#: ⭐⭐ W41-T05 / CONTRACT C3 — THE DESCRIPTION EVERY ROUTE COLUMN IS BORN WITH. +#: +#: `shared_overlay.mint_field` refuses a tenant-wide column that has no description, and the +#: reason is the rule: this column lands in the field list of people who did not make it and +#: cannot ask what its integers mean. W41-T04 recorded the gap here by name (*"a `note` on the +#: definitions minted by `routes_customers::route_order_write`"*) as one of the two call sites +#: that had to start supplying one before the triple could become a hard refusal. +#: +#: ⚠ IT IS A DEFAULT, NOT A LOCK. `route_order_rename` writes a user-authored `note` onto the +#: same definition and an EMPTY STRING is how a person CLEARS one, so the write door below +#: carries the stored value forward by PRESENCE rather than by truthiness. A truthy test would +#: resurrect this sentence on the next re-solve for anybody who had deliberately deleted it. +ROUTE_FIELD_NOTE = ("The visit order of a saved route. Each number is that customer's position " + "in the day, counting from 1, and it changes only when somebody plans the " + "route again.") + +#: ⛔⛔ THE TOPIC A FIELD GRANT IS NAMED UNDER, AND IT IS **NOT** THE STORE BUCKET. Two namespaces +#: meet on this column and they are spelled differently: +#: +#: the STRATUM lives at `shared_overlay.bucket(customer_data.TABLE_KEY)` +#: = `customer_table_workspace__shared` +#: the GRANT is named `shares.field_oid(, key)` +#: = `customer_data:` +#: +#: `perm_scope.hidden_keys(user, module, fields)` hands its `module` argument straight through to +#: `field_grant_hidden`, which builds the oid from it. On a `ut_*` database the module argument IS +#: the table key, so W38-T16 never had to tell them apart; on a REGISTRY topic they differ, and a +#: door that claims the grant under the bucket name writes a record the wall will never look for. +#: MEASURED, not reasoned: the first run of this ticket's gate did exactly that, and user B was +#: refused a column that had been shared with them through the real share door, with a 200 at +#: every step ([[one-question-two-normalizers]]). +SHARE_TOPIC = MODULE + + +def _route_slug(label): + """A stable store key from a human label. Lower case, non-alphanumerics collapsed to `_`.""" + import re as _re + slug = _re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") + return f"{ROUTE_KEY_PREFIX}{slug[:48]}" if slug else "" + + +def _occupied(defs, base_fields=()): + """The keys and case-folded labels a new route column must not land on.""" + occupied_keys = {str(key) for key in (defs or {})} + occupied_labels = set() + for field in list((defs or {}).values()) + list(base_fields or ()): + if not isinstance(field, dict): + continue + label = " ".join(str(field.get("label") or "").split()).casefold() + if label: + occupied_labels.add(label) + key = str(field.get("key") or "").strip() + if key: + occupied_keys.add(key) + return occupied_keys, occupied_labels + + +def _next_route_label(defs, base_fields=()): + """Return the next human route-field label without colliding with a visible field. + + ⭐ OWNER ITEM 5 (2026-08-23) — THE WORD IS "ROUTE", NOT "DESTINATION". Owner: *"Instead of + calling it 'Destination 1' etc. for when we saved the route to a field, we just call it + 'Route 1' so it's Route 1, Route 2, Route 3."* A saved column holds the ORDER of a whole day, + so "Destination 1" read as the first stop rather than as the first route, which is exactly + backwards from what the numbers in it mean. + + ⛔ NO MIGRATION, AND THAT IS DELIBERATE. Columns already minted as `Destination N` keep their + label: rewriting a tenant-wide column name that other people's saved views point at is not a + rename this door was asked for. The owner renames one through `route_order_rename` below, per + owner item 6 of the same instruction. + + ⚠ THE KEY IS STILL DERIVED FROM THE LABEL, so the default now slugs to `route_route_1`. The + redundant segment is NOT tidied away: `ROUTE_KEY_PREFIX` is what keeps a route column out of + the namespace this database owns, and stripping it here would make the labels "Route 1" and + "1" collide on one key. The key is never on screen. + """ + occupied_keys, occupied_labels = _occupied(defs, base_fields) + n = 1 + while True: + label = f"Route {n}" + if label.casefold() not in occupied_labels and _route_slug(label) not in occupied_keys: + return label + n += 1 + + +def _clean_route_depot(raw): + """The optional origin stored once on a route-order definition, never on a customer cell.""" + if raw is None: + return None + if not isinstance(raw, dict): + raise err(400, "bad_depot", "a depot is an address with latitude and longitude, or null") + address = " ".join(str(raw.get("address") or "").split())[:200] + lat, lon = raw.get("lat"), raw.get("lon") + if (not address or isinstance(lat, bool) or isinstance(lon, bool) or + not isinstance(lat, (int, float)) or not isinstance(lon, (int, float))): + raise err(400, "bad_depot", "a depot needs an address and numeric latitude and longitude") + lat, lon = float(lat), float(lon) + if not (math.isfinite(lat) and math.isfinite(lon) and abs(lat) <= 90 and abs(lon) <= 180): + raise err(400, "bad_depot", "the depot latitude or longitude is outside the map") + return {"address": address, "lat": lat, "lon": lon} + + +def _route_defs(session: Session): + """This topic's route-order columns MINUS the ones this session was not granted. + + ⛔ THE WALL IS THE SAME ONE THE GRID USES, NOT A SECOND OPINION. `hidden_keys` is where T16 + put the per-field grant check, so filtering on it here means the listing, the grid contract + and the row payload agree by construction. A column a reader cannot see must not appear here + either: the done-when says *does not see the field at all*, and a picker that names a column + whose values are withheld has already leaked its existence. + """ + import aios_grid + import core.perm_scope as perm_scope + + all_defs = shared_fields(st=session.runtime) or {} + defs = {k: v for k, v in all_defs.items() + if isinstance(v, dict) and v.get("kind") == ROUTE_KIND} + if not defs: + return {} + hide = perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs), + st=session.runtime) + return {k: v for k, v in defs.items() if k not in hide} + + +def _own_route_fork(session: Session, key: str): + """This caller's PRIVATE copy of a route column, if a per-user write ever forked one. + + ⛔⛔ THE FORK IS REAL, AND IT IS WHAT BROKE DELETE. Traced through `grid_events.field_upsert` + rather than assumed: `shared_field` is `bool(shared_prior.get('custom'))` and a route + definition carries no `custom` key, so the shared branch does not take it; the key is neither + `custom_` nor `measure_`; it IS in the merged contract, so the `key in field_by_key` branch + does — and that branch's own body is what writes `note`, `format` and `agg`, landing them in + THIS USER'S field definitions through `TableStore.save_field`. `_merge_shared_fields` then + SKIPS the tenant-wide definition, because a key the contract already declares wins, and from + that moment the person is reading a private copy of a shared column. + + ⚠ WHICH DOORS STILL FORK, AS OF 2026-08-23, because "the fork is fixed" would be too broad. + The Description and the Name are CLOSED: owner item 6 routed `onNote` and `onRename` to + `route_order_rename` below. The Edit-field pane's **Format** row (`onFormat`, unconditional) + and its **Summary** row (`onAggregate`, whose `isUserTable` arm is false on this registry + topic) both still reach `saveField`, so either one still mints a fork. They are left open on + purpose: closing them needs this door to accept `format` and `agg`, which is a wider change + than the delete the owner reported. This function is what keeps that recoverable rather than + permanent. + + ⛔ WHY THAT MADE THE DELETE 404 RATHER THAN MERELY MISBEHAVE. The first delete found the + tenant-wide definition, dropped it and answered 200 — and the column came straight back on the + next read, because the fork was still declaring it. The second attempt found nothing in + `shared_fields` and answered *"that column is not a route order column on this database"* + about a column sitting on screen. Owner, 2026-08-23: *"I can't even delete the Field route + now?"* Both halves are answered here: a fork is FOUND, and `route_order_delete` drops it WITH + the definition rather than leaving it to redeclare the column. + + ⚠ `kind` IS THE TEST, NEVER MERE PRESENCE. `TableStore.workspace` merges the tenant-wide + column summary over this stratum and will mint a bare `{'agg': ...}` entry for a key the user + has never touched (W36-T25, whose own comment says it must be able to CREATE an entry). + Reading presence would call that stub a fork and scrub a column nobody had forked. + """ + try: + ws = _customer_table(session).workspace(session.uname, consume_corrections=False) + except Exception: # noqa: BLE001 + return None + entry = (ws.get("fields") or {}).get(key) + if not isinstance(entry, dict) or entry.get("kind") != ROUTE_KIND: + return None + return entry + + +@router.get("/customers/route-order") +def route_order_list(session: Session = Depends(module_gate(MODULE))): + """The route-order columns this session may see, with the fingerprint each was solved from. + + ⭐⭐ THE FINGERPRINT IS WHY THIS ROUTE EXISTS RATHER THAN THE CLIENT READING `fields`. + Staleness is DERIVED, never stored: what is written down is the INPUT FINGERPRINT the numbers + were produced from, so *is this order still current* is a question asked at READ time against + what is on screen NOW, and can never itself be out of date. A stored `stale: true` is a fact + about a moment that has already passed. + """ + out = [] + for key, defn in sorted(_route_defs(session).items()): + route = defn.get("route") if isinstance(defn.get("route"), dict) else {} + out.append({ + "key": key, + "label": defn.get("label") or key, + "inputsHash": str(route.get("inputsHash") or ""), + "roundTrip": bool(route.get("roundTrip")), + "startPid": route.get("startPid"), + "stops": route.get("stops"), + "depot": route.get("depot") if isinstance(route.get("depot"), dict) else None, + "solvedAt": route.get("solvedAt") or "", + "solvedBy": defn.get("createdBy") or "", + # Who may re-solve it. The same creator-or-admin wall the write door enforces, said on + # the way out so the client can grey the control instead of discovering a 403. + "mine": bool(session.admin + or str(defn.get("createdBy") or "") == session.uname), + }) + # ⭐⭐ OWNER ITEM 5 — THE DEFAULT NAME IS ALLOCATED **HERE**, NEVER ON THE CLIENT. + # + # The panel prompts for a route name and pre-fills it. Computing that pre-fill from the + # `fields` list above would compute it from the columns this session may SEE: `_route_defs` + # drops every route column the per-field wall hides, so a user with no grant on + # `route_route_1` would be offered "Route 1" as their default, send it, and be refused + # `field_key_taken` on a name the app itself put in the box. Allocated over the WHOLE shared + # stratum, the suggestion can never name a column that already exists. + # + # ⚠ IT LEAKS NOTHING. What travels is the first FREE name, which is a fact about absence. + import aios_grid + return {"fields": out, + "nextLabel": _next_route_label(shared_fields(st=session.runtime) or {}, + aios_grid.FIELDS)} + + +@router.post("/customers/route-order") +def route_order_write(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Create (or re-solve) a tenant-wide route-order column and fill it, in ONE call. + + `{label?, field?, ranks: {"": }, inputsHash, roundTrip?, startPid?, depot?}` + + When `label` is omitted for a new column, the door allocates the next unused `Destination N` + label. This keeps route creation one-click while preserving unique, readable field names. + + ⛔ THE DEFINITION IS WRITTEN FIRST AND THE GRANT CLAIMED SECOND — inside + `shared_overlay.mint_field` since W41-T05, which is where the `set_grants` call a reader looks + for in this function now lives. It is `patch_shared_cell`'s + order and it is deliberate: the window where a column is MARKED and UNCLAIMED fails CLOSED + (governed, nobody granted, so only an admin and the creator see it, and an admin can share + it). The other order would leave a grant record pointing at nothing. + + ⛔ RE-SOLVING IS CREATOR-OR-ADMIN, WHICH CREATING IS NOT. Writing these numbers changes what + every account in the workspace reads, at once, for the whole cohort. That is the wall + `routes_tables.delete_shared_field` already applies to the destructive half of this stratum, + and a new door does not get to inherit the loose half of an asymmetry somebody has flagged. + A permitted teammate READS the numbers; they do not silently re-plan somebody's day. + """ + from core import shared_overlay + import core.perm_scope as perm_scope + from routes_grid import MAX_BULK_ROWS + + body = body if isinstance(body, dict) else {} + label = " ".join(str(body.get("label") or "").split())[:120] + import aios_grid + defs = shared_fields(st=session.runtime) or {} + requested_key = str(body.get("field") or "").strip() + if requested_key: + key = requested_key + elif label: + key = _route_slug(label) + else: + label = _next_route_label(defs, aios_grid.FIELDS) + key = _route_slug(label) + if not key: + raise err(400, "bad_request", "a name is required for the route order column") + if not key.startswith(ROUTE_KEY_PREFIX): + raise err(400, "bad_field_key", + f"a route order column's key starts with '{ROUTE_KEY_PREFIX}', so it cannot " + f"collide with a column this database already owns") + if key in {f.get("key") for f in aios_grid.FIELDS}: + raise err(400, "field_key_taken", + "this database already has a column with that key") + + existing = defs.get(key) if isinstance(defs.get(key), dict) else None + if existing is not None: + # ⛔⛔ A REFUSAL MUST NOT DESCRIBE A COLUMN THE CALLER CANNOT SEE. The two refusals below + # name the column's KIND and its CREATOR, which is exactly the information the field wall + # exists to withhold: a stranger who guesses the key would otherwise learn that a route + # order exists on this database and who planned it. `routes_shares._can_see_object` makes + # the same choice for the same reason (a non-grantee gets the answer a non-existent id + # gets). The name is already taken either way, so the honest refusal says only that. + if key in perm_scope.hidden_keys( + session.user, MODULE, + _merge_shared_fields(list(aios_grid.FIELDS), defs), st=session.runtime): + raise err(400, "field_key_taken", + "that column name is already in use on this database") + if existing.get("kind") != ROUTE_KIND: + raise err(400, "not_a_route_column", + "that column is shared but it is not a route order column, so re-solving " + "it would overwrite values this door did not write") + owner = str(existing.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a route order can be re-solved by the person who planned it or by an " + f"administrator. This one was planned by {owner or 'somebody else'}, and " + f"re-solving it would change the visit numbers for every account at once") + + if "depot" in body: + depot = _clean_route_depot(body.get("depot")) + else: + prior_route = (existing or {}).get("route") if isinstance(existing, dict) else {} + depot = prior_route.get("depot") if isinstance(prior_route, dict) else None + + raw = body.get("ranks") + if not isinstance(raw, dict) or not raw: + raise err(400, "bad_ranks", 'expected {ranks: {"": }}') + # ⛔ REPORTED, NEVER TRUNCATED (standing rule 1's second sentence, and `MAX_BULK_ROWS`' own + # note). The ceiling is `routes_grid`'s so there is ONE of them, not two that drift. + if len(raw) > MAX_BULK_ROWS: + raise err(400, "too_many_rows", + f"at most {MAX_BULK_ROWS} records per route; this one carried {len(raw)}") + + pool = allowed_pids(session) + ranks, not_in_pool, bad_value = {}, [], [] + for raw_pid, value in raw.items(): + try: + pid = int(raw_pid) + except (TypeError, ValueError): + not_in_pool.append(str(raw_pid)[:40]) + continue + # ⚠ THE POOL IS THE WALL, and it is the SAME predicate the read path applies + # (`apply_row_scope` inside `allowed_pids`), so a caller cannot number a record they + # could not be shown, including one in the other business unit. + if pid not in pool: + not_in_pool.append(str(raw_pid)[:40]) + continue + # ⛔ A BOOL IS AN `int` IN PYTHON and `True` would store as a visit number 1. Excluded by + # name rather than by hoping nobody sends one. + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + bad_value.append(str(raw_pid)[:40]) + continue + ranks[pid] = value + + # ⛔⛔ A PARTLY HONOURED ROUTE IS NOT A ROUTE, AND THIS REFUSES THE WHOLE CALL RATHER THAN + # WRITING THE PART IT COULD. Caught by this ticket's own gate: a body carrying one record + # outside the caller's book still produced a clean 1..N over what was left, so the door minted + # a permanent tenant-wide column and filled it with a SHORTER route than the one that was + # solved. Every number in it was plausible and the day was wrong. + # ⚠ REPORTED, WITH THE SAMPLE, which is standing rule 1's second sentence: the refusal names + # what it could not take, so the caller can fix it rather than guess. + if not_in_pool: + raise err(400, "rows_not_in_your_book", + f"{len(not_in_pool)} of those records are not in your book, so the route " + f"cannot be written as it was solved. First few: " + f"{', '.join(sorted(not_in_pool)[:5])}") + if bad_value: + raise err(400, "rows_not_a_visit_number", + f"a visit number is a whole number from 1 upwards; {len(bad_value)} records " + f"carried something else. First few: {', '.join(sorted(bad_value)[:5])}") + if not ranks: + raise err(400, "no_rows_in_your_book", + "none of those records are in your book, so there is nothing to number") + # ⛔ A ROUTE IS A SEQUENCE, SO THE NUMBERS ARE 1..N WITH NO REPEAT AND NO GAP. A body that + # arrives truncated, doubled or partly applied would otherwise land as a plausible half + # route: every row carrying a number, and the day in the wrong order. + seq = sorted(ranks.values()) + if seq != list(range(1, len(seq) + 1)): + raise err(400, "not_a_sequence", + f"a route order is the numbers 1 to {len(seq)}, each used once. This one " + f"carried {len(seq)} records numbered up to {seq[-1]} with " + f"{len(seq) - len(set(seq))} repeated") + + stamp = time.strftime("%Y-%m-%d %H:%M") + defn = { + "key": key, + "label": label or (existing or {}).get("label") or key, + # ⭐⭐ OWNER ITEM 7 (2026-08-23) — THE SAVED ROUTE IS A **NUMBER** COLUMN. + # + # Owner: *"the Route field once saved should be saved as a number Field. Make sure we can + # edit it, BUT also make sure that none of the number can be a duplicate, so its like a + # number order field."* It used to be a `select` whose declared choices were the strings + # "1".."N", which sorted and rendered as a list rather than as an order and forced a + # re-solve to widen the vocabulary before a stop could be numbered past N. + # + # ⛔ UNIQUENESS IS NOT A PROPERTY OF THE TYPE, SO IT LIVES AT THE WRITE DOOR. `int` has no + # "no duplicates" flag anywhere in this contract; `_patch_route_ranks` in `patch_customer` + # is what keeps the column a permutation, by SWAPPING rather than by refusing. + # + # ⚠ EXISTING COLUMNS ARE NOT MIGRATED, they are UPGRADED BY RE-SOLVE. This dict is + # rebuilt whole on every write, so the first re-solve of a `Destination N` column turns it + # into a number column; one nobody re-solves keeps working as the select it was, and the + # write door reads the KIND rather than the type, so both edit identically. + "type": "int", + "source": "overlay", + "shared": True, + "kind": ROUTE_KIND, + # ⭐⭐ W41-T05 — THE DESCRIPTION, CARRIED BY **PRESENCE** AND NOT BY TRUTHINESS. + # `ROUTE_FIELD_NOTE` says why the column cannot be born without one and why `""` has to + # survive a re-solve: `route_order_rename` clears a description by storing exactly that, + # and `existing.get("note") or ROUTE_FIELD_NOTE` would hand the default straight back to + # the person who had just deleted it. `"note" in existing` is the question with an + # answer; truthiness is a different question wearing the same shape. + "note": (existing["note"] if isinstance(existing, dict) and "note" in existing + else ROUTE_FIELD_NOTE), + # ⭐⭐ OWNER, 2026-08-23 — A NEW ROUTE READS AS **PRIVATE**, NEVER "Shared with everyone". + # + # Owner: *"Right now the Route is 'Shared with everyone' in terms of the Field status when + # i check the Route Field. It should always default to private first."* + # + # ⛔ A CLASSIFICATION FIX, NOT A TIGHTENING, AND THE DIFFERENCE IS THE WHOLE POINT. This + # column was ALREADY private in the only sense that governs a reader: `FIELD_GRANT_MARK` + # plus the empty-entry grant claimed below means creator-and-admin and nobody else. What + # was wrong was the WORD ON SCREEN. `FieldsHidePanel` sections the field list by + # `types.fieldEditMode`, which is `cleanFieldPermissions(field.permissions, + # "collaborative")` — so a definition carrying NO permissions bag fell to that fallback + # and was filed under "Shared with everyone" while being shared with nobody at all. + # + # ⚠ AND IT MOVES NO WALL, CHECKED RATHER THAN REASONED. `grid_events._may_edit_field_value` + # short-circuits on `definition.get('shared') or definition.get('granted')` BEFORE it reads + # `stored_permissions`, so the creator's own rank edits and `_patch_route_ranks`' swap are + # decided by `_field_share_role` and not by this bag; the client twin `mayEditField` takes + # the same branch in the same order. `field_permissions.migrate_legacy_fields` only ever + # rewrites a PER-USER field carrying `custom: True`, which a route definition is not, so it + # leaves this key alone. `fieldEditMode` has exactly one other consumer: none. + # + # ⛔ NO MIGRATION, AND IT IS THE SAME DELIBERATE CHOICE `_next_route_label` MAKES ABOUT + # `Destination N`. This dict is rebuilt whole on every write, so a column minted before + # today gains the bag on its next RE-SOLVE and not one moment sooner — until then it + # keeps reading "Shared with everyone" in the Hide fields panel while being shared with + # nobody. Backfilling every stored route definition on a read is a write nobody asked + # for, on a tenant-wide stratum, triggered by opening a page; re-solving is one click and + # it is the click the owner is already making. Stated here rather than left to be + # rediscovered as "the fix did not work". + "permissions": {"edit": "personal"}, + # ⭐⭐ THE PER-FIELD GRANT MARKER (T16). It is an EXPLICIT write-once declaration and it is + # what makes the wall fail CLOSED: a grant wall has the opposite absence-polarity to a + # deny wall, so keying visibility on "does a grant record exist" would publish this column + # to the whole tenant on one unreadable read of `object_shares`. Stamped, never inferred. + perm_scope.FIELD_GRANT_MARK: True, + "createdBy": (existing or {}).get("createdBy") or session.uname, + # ⭐ THE FINGERPRINT RIDES THE DEFINITION, ONCE. One `planRoute` run solves the whole + # cohort, so this string is identical for every record in it; per row it would be the same + # value written N times, and `shared_overlay._value` raises on a dict anyway. + "route": { + "inputsHash": str(body.get("inputsHash") or "")[:64], + "roundTrip": bool(body.get("roundTrip")), + "startPid": int(body["startPid"]) if isinstance(body.get("startPid"), int) + and not isinstance(body.get("startPid"), bool) else None, + "stops": len(ranks), + "depot": depot, + "solvedAt": stamp, + }, + } + # ⭐⭐ W41-T05 / CONTRACT C3 — A BIRTH GOES THROUGH `mint_field`, A RE-SOLVE THROUGH + # `put_field`, AND THE SPLIT IS THE WHOLE FIX. C3 names this door as the one bypass in the + # tree: it reached around the column menu into the raw definition writer because + # `aios_grid._field_extras` would have stripped the `route` bag. W41-T04 widened the + # allowlist to carry that bag, so the bypass buys nothing any more, and `mint_field` is the + # entrance that REFUSES a column born without a creator, a grant list and a description. + # + # ⛔⛔ EVERY MEMBER OF THE TRIPLE STAYS IN THE LITERAL ABOVE, AND THAT IS NOT REDUNDANCY. + # `mint_field` stamps `createdBy`, `granted` and `note` itself, but it runs on the MINT ONLY; + # a re-solve goes to `put_field`, whose allowlist makes an omitted key CLEAR the stored one + # ("an allowlisted key follows the caller exactly"). Leaning on the mint for the mark would + # give a perfect create and a re-solve that silently UNMARKS the column and re-opens the very + # leak this ticket closes. The literal is what makes both paths land the same definition. + # + # ⚠ THE EMPTY GRANT LIST IS THE POINT, and `mint_field` keeps it: `set_grants` stores a record + # with an owner and no entries, so "shared with nobody" is a STORED fact and a different one + # from "never shared". Without the owner the column is unmanageable — `may_administer` fails + # closed on an ownerless record, so nobody could ever share it. The definition still lands + # BEFORE the claim (inside `mint_field`, same order, same reason): a marked and unclaimed + # column is admin-and-creator only, which is recoverable; the other order is not. + if existing is None: + try: + shared_overlay.mint_field( + _shared_key(), key, defn, + created_by=defn["createdBy"], grants=[], description=defn["note"], + grant_topic=SHARE_TOPIC, st=session.runtime) + except ValueError as exc: + # ⛔ `mint_field` REFUSES rather than overwriting, and only two things can raise here. + # The triple is supplied one line up, so the reachable refusal is "that key already + # exists" — which `existing` above said it did not. The two disagree in exactly one + # case: `existing` is TYPE-checked (`isinstance(..., dict)`) while `is_shared` is a + # bare membership test, so a junk non-dict entry parked under the key reads as absent + # to one and present to the other. It answers with the taken-name 400 this door + # already has rather than a 500, and anything else re-raises rather than being + # relabelled as a name collision it is not. + if not shared_overlay.is_shared(_shared_key(), key, st=session.runtime): + raise + raise err(400, "field_key_taken", + "that column name is already in use on this database") from exc + else: + shared_overlay.put_field(_shared_key(), key, defn, st=session.runtime) + + # ⛔ AND THE RECORDS THAT LOST THEIR NUMBER ARE CLEARED. A re-solve over a SMALLER cohort + # would otherwise leave the previous run's ranks sitting on the records that dropped out — + # plausible integers, from a route nobody is driving. `""` is the house spelling of an empty + # overlay cell (`rows_from_pool` defaults an absent one to exactly that), so this needs no new + # vocabulary and no tombstone nobody else reads. + stale = {} + for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): + if not isinstance(cells, dict) or cells.get(key) in (None, ""): + continue + try: + gone = int(raw_pid) + except (TypeError, ValueError): + continue + if gone not in ranks: + stale[gone] = {key: ""} + written = shared_overlay.put_rows( + _shared_key(), {**{p: {key: str(v)} for p, v in ranks.items()}, **stale}, + st=session.runtime) + + out = {"ok": True, "field": key, "label": defn["label"], "type": "int", + "stops": len(ranks), "cleared": len(stale), + "rows_written": len(written), "inputsHash": defn["route"]["inputsHash"], + "depot": defn["route"]["depot"], + "solvedAt": stamp} + return out + + +@router.delete("/customers/route-order/{field_key}") +def route_order_delete(field_key: str, + session: Session = Depends(module_gate(MODULE))): + """Remove a Destination column and every visit number in it. + + ⛔⛔ WHY THIS DOOR HAD TO EXIST. `route_order_write` mints a TENANT-WIDE column and nothing + could ever remove it: the grid's own Delete is offered for a per-user `custom_` column + (`menuField.custom && !menuField.shared`) or for a `ut_*` definition field, and a route order + is neither — it is a SHARED column on a registry topic. So `Destination 1` was permanent for + the whole workspace, which is [[reachable-is-not-the-same-as-built]] from the other end: + `shared_overlay.drop_field` was complete and correct and no route on this topic called it. + + ⛔ CREATOR-OR-ADMIN, the same wall `routes_tables.delete_shared_field` applies and for the + same reason: writing a cell changes a value, dropping the column deletes that value for every + account at once. It is deliberately NOT the looser wall `route_order_write` uses for CREATE. + + ⛔ AND A CALLER WHO CANNOT SEE THE COLUMN GETS THE ANSWER A NONEXISTENT KEY GETS. The refusals + below would otherwise teach a stranger that a route order exists on this database and who + planned it, which is exactly what the per-field wall withholds — the choice already argued at + `route_order_write`'s `field_key_taken` and in `routes_shares._can_see_object`. + """ + from core import shared_overlay + import core.perm_scope as perm_scope + import core.table_store as table_store + import aios_grid + + key = str(field_key or "").strip() + all_defs = shared_fields(st=session.runtime) or {} + defn = all_defs.get(key) if isinstance(all_defs.get(key), dict) else None + # ⭐⭐ OWNER, 2026-08-23 — A PRIVATE FORK IS ALSO A COLUMN TO DELETE. See `_own_route_fork` + # above: once any per-user write has forked this key, the fork is what the person is looking + # at and it OUTLIVES a drop of the tenant-wide definition. Answering 404 about a column that + # is on screen is exactly the refusal the owner hit. + fork = _own_route_fork(session, key) + subject = defn or fork + unknown = err(404, "unknown_field", + "that column is not a route order column on this database") + if not subject: + raise unknown + # The wall FIRST, so a hidden column is indistinguishable from an absent one. + # ⚠ ASKED ABOUT THE TENANT-WIDE DEFINITION ONLY, and that is not a hole. The wall reads + # `FIELD_GRANT_MARK` off the MERGED contract; with the definition already gone there is no + # marked column left for it to hide, and a fork lives in this caller's OWN stratum, which no + # grant has ever governed. Not asking when there is nothing to ask about beats asking of a + # contract that no longer carries the key and reading the empty answer as "not hidden". + if defn and key in perm_scope.hidden_keys( + session.user, MODULE, + _merge_shared_fields(list(aios_grid.FIELDS), all_defs), st=session.runtime): + raise unknown + # ⛔ KIND-CHECKED, NOT PREFIX-CHECKED. `ROUTE_KEY_PREFIX` is a naming convention and a + # convention is not a declaration (the comment on `ROUTE_KIND` says so); a door that deleted + # by prefix would happily drop a shared column somebody else's feature owns. + if subject.get("kind") != ROUTE_KIND: + raise err(400, "not_a_route_column", + "that column is shared but it is not a route order column, so this door will " + "not remove it") + owner = str(subject.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a route order column can be removed by the person who planned it or by an " + f"administrator. This one was planned by {owner or 'somebody else'}, and " + f"removing it would delete the visit numbers for every account at once") + + dropped = bool(defn) and shared_overlay.drop_field(_shared_key(), key, st=session.runtime) + # ⛔⛔ AND THE FORK GOES IN THE SAME CALL. Dropping only the tenant-wide definition is what + # made the first delete look like it had worked and then undo itself: the fork still declares + # the column, `_merge_shared_fields` still yields to it, and the next paint brings it back. + # `delete_field` scrubs this user's overlay values under the key too, which is right: the + # tenant-wide cells went with `drop_field` above, and a private leftover would resurface under + # whatever column later took the key. + if fork is not None: + try: + table_store.make(_shared_key(), st=session.runtime).delete_field(session.uname, key) + except Exception: # noqa: BLE001 + # The definition is already gone, so a failed fork scrub must not turn a delete that + # succeeded into a 500. Worst case the column lingers for this one account until its + # next write, which is recoverable; a 500 over completed work is not. + pass + # ⭐⭐ THE GRANT DIES WITH THE COLUMN. `route_order_write` claims + # `shares.field_oid(SHARE_TOPIC, key)` on create, so skipping this would leave a grant record + # pointing at nothing — a ghost in every receiver's "Shared with me" that 404s on open, and + # worse, one that silently re-arms on the next column to take the key, because `drop_field` + # scrubs the cells precisely so the key CAN be re-used. + # ⚠ Same order and same tolerance as `routes_tables.delete_shared_field`: the column is + # already gone, so a failed release must not turn a completed delete into a 500. + try: + import core.shares as shares + shares.drop_objects([("field", shares.field_oid(SHARE_TOPIC, key))], st=session.runtime) + except Exception: # noqa: BLE001 + pass + return {"ok": True, "field": key, "dropped": bool(dropped or fork is not None)} + + +@router.patch("/customers/route-order/{field_key}") +def route_order_rename(field_key: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Rename a route order column, or write its description. `{label?, note?}`. + + ⭐⭐ OWNER ITEM 6 (2026-08-23) — "Edit field" MUST BE ABLE TO RENAME THIS COLUMN. + + A route order is a SHARED column on the customers REGISTRY topic. The grid's Edit-field pane + offers a Name box only when the host supplies `onRename`, and the host supplies it only for + `isUserSchemaField` — a per-user `custom_` column, or a `ut_*` definition field. A route order + is neither, so the pane showed a DISABLED Name box reading *"A source field keeps its name and + type from the data source"* about a column the person had created themselves an hour earlier. + That is the same shape as owner item 3's missing Delete, one stratum over. + + ⛔⛔ THE KEY IS FROZEN. `_route_slug` derives the store key from the label AT CREATION, and + only there. Re-slugging on a rename would leave every written visit number sitting under the + old key in `shared_overlay`, every saved view's `colId` pointing at a column that no longer + exists, and every field grant naming an oid nobody can reach — a rename that looks perfect on + a fresh column and quietly empties a used one. The label moves; nothing else does. + + ⛔ CREATOR-OR-ADMIN, and a caller who cannot SEE the column gets the answer a nonexistent key + gets. Both walls are `route_order_delete`'s, verbatim and for its reasons: this changes what + every account in the workspace reads, and a refusal that said "forbidden" would confirm to a + stranger that a route order exists here and who planned it. + + ⛔⛔ THE DESCRIPTION RIDES THIS DOOR TOO, AND IT HAD TO. The Edit-field pane writes a + description through `onNote`, which is `saveField` — a PER-USER `field_upsert`. Traced through + `grid_events`: a route key is not `custom`-marked in the shared stratum, so the `shared_field` + branch does not take it; it IS in the merged contract, so the `key in field_by_key` branch + does, and it stores `{**base, note}` in THIS USER'S field definitions. `_merge_shared_fields` + then skips the shared definition, because a key the contract already declares wins — so that + user is left reading a private copy of a tenant-wide column, frozen at the label it had when + they typed the description, and a later rename through this door is invisible to them. + Pre-existing, and unreachable enough to have gone unnoticed; owner item 6 makes that pane the + place people go for route columns, so it stops being unreachable on the same day. + """ + from core import shared_overlay + import core.perm_scope as perm_scope + import aios_grid + + body = body if isinstance(body, dict) else {} + key = str(field_key or "").strip() + label = " ".join(str(body.get("label") or "").split())[:120] + # ⚠ `"note" in body` AND NOT a truthiness test: an empty string is how a description is + # CLEARED, and a door that read it as "unchanged" would leave one nobody can delete. + note = str(body.get("note") or "")[:2000] if "note" in body else None + if not label and note is None: + raise err(400, "bad_request", "a route order column needs a name") + if "label" in body and not label: + raise err(400, "bad_request", "a route order column needs a name") + + all_defs = shared_fields(st=session.runtime) or {} + defn = all_defs.get(key) if isinstance(all_defs.get(key), dict) else None + unknown = err(404, "unknown_field", + "that column is not a route order column on this database") + if not defn: + raise unknown + if key in perm_scope.hidden_keys( + session.user, MODULE, + _merge_shared_fields(list(aios_grid.FIELDS), all_defs), st=session.runtime): + raise unknown + if defn.get("kind") != ROUTE_KIND: + raise err(400, "not_a_route_column", + "that column is shared but it is not a route order column, so this door will " + "not rename it") + owner = str(defn.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a route order column can be renamed by the person who planned it or by an " + f"administrator. This one was planned by {owner or 'somebody else'}, and its " + f"name is what every account in the workspace reads") + + # ⚠ THE COLLISION CHECK LOOKS AT LABELS AND NOT AT KEYS, because the key is frozen: this + # rename can never take another column's key, only its NAME. Two columns wearing one name on + # the same grid is the confusion `_next_route_label` exists to prevent at creation, so the + # rename door refuses it too. The column's OWN current label is excluded, so re-saving an + # unchanged name is a no-op rather than a refusal. + _keys, occupied_labels = _occupied( + {k: v for k, v in all_defs.items() if k != key}, aios_grid.FIELDS) + if label and label.casefold() in occupied_labels: + raise err(400, "field_label_taken", + "this database already has a column with that name") + + patch = dict(defn) + if label: + patch["label"] = label + if note is not None: + patch["note"] = note + shared_overlay.put_field(_shared_key(), key, patch, st=session.runtime) + return {"ok": True, "field": key, "label": patch.get("label") or key, + "note": patch.get("note") or ""} diff --git a/api/routes_grid.py b/api/routes_grid.py index 75d25825a1b4de8b88d0f9ae8580a02b64282dce..54148608a4d59551ad9416d1c8016b8bdcbd1a18 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -1,1092 +1,1376 @@ -"""routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c). - -`POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the -Streamlit host receives through the component value slot, unchanged — and runs them through the -same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of -EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated -anything here would be a second wall to keep in step, and the two would drift on the first change. - -DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's -`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends -its recent 24-event window on every emit inside one page session. A stateless API has no such -dict, and inventing a per-user server-side one would be exactly the resident per-tenant state -EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole -purpose — a batch that repeats an id processes it once), and a genuinely replayed request is -handled by the operations being idempotent. The one event where a replay is observable is -`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the -membership is unchanged, but the toast count repeats). Noted rather than papered over; a -server-side idempotency key belongs with D2's session mirror (C-2). - -STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises -`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read -back. A 200 over a write that evaporated is the failure this rule exists to prevent. -""" -import datetime as dt -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, module_gate, perms, require_session - -router = APIRouter(prefix="/api/v1") - -MODULE = "customer_data" - -#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a -#: legitimate client — refuse it rather than doing 500 store writes on one request. -_MAX_EVENTS = 24 - - -def _ctx(session: Session, fields, pids, **kw): - from core import grid_events - import aios_grid - import core.perm_scope as perm_scope - - # Wave 16 C-TOPIC: the ctx is TOPIC-SHAPED. The product scope swaps all three of the - # things a write is validated against — the module the permission wall reads, the canonical - # contract the hidden-field closure runs over, and the TABLE OPS the write lands in. - # Getting any one of them from the other topic is the "validated against the wrong field - # contract" near-miss the wave-15 routes_products header warned about. - scope_key = str(kw.get("scope_key") or "") - if scope_key == "product": - import modules.product_data as pd - from routes_products import MODULE as _PMOD, pd_fields - +"""routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c). + +`POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the +Streamlit host receives through the component value slot, unchanged — and runs them through the +same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of +EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated +anything here would be a second wall to keep in step, and the two would drift on the first change. + +DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's +`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends +its recent 24-event window on every emit inside one page session. A stateless API has no such +dict, and inventing a per-user server-side one would be exactly the resident per-tenant state +EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole +purpose — a batch that repeats an id processes it once), and a genuinely replayed request is +handled by the operations being idempotent. The one event where a replay is observable is +`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the +membership is unchanged, but the toast count repeats). Noted rather than papered over; a +server-side idempotency key belongs with D2's session mirror (C-2). + +STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises +`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read +back. A 200 over a write that evaporated is the failure this rule exists to prevent. +""" +import datetime as dt +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, module_gate, perms, require_session + +router = APIRouter(prefix="/api/v1") + +MODULE = "customer_data" + +#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a +#: legitimate client — refuse it rather than doing 500 store writes on one request. +_MAX_EVENTS = 24 + + +def _ctx(session: Session, fields, pids, **kw): + from core import grid_events + import aios_grid + import core.perm_scope as perm_scope + + # Wave 16 C-TOPIC: the ctx is TOPIC-SHAPED. The product scope swaps all three of the + # things a write is validated against — the module the permission wall reads, the canonical + # contract the hidden-field closure runs over, and the TABLE OPS the write lands in. + # Getting any one of them from the other topic is the "validated against the wrong field + # contract" near-miss the wave-15 routes_products header warned about. + scope_key = str(kw.get("scope_key") or "") + if scope_key == "product": + import modules.product_data as pd + from routes_products import MODULE as _PMOD, pd_fields + module, canonical = _PMOD, pd_fields(consolidated=True, st=session.runtime) kw.setdefault("table", pd.table_ops(session.runtime)) hidden = perm_scope.hidden_keys(session.user, module, canonical, st=session.runtime) - elif scope_key.startswith("ut_"): - # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is - # `user_tables.may_open`, already applied by the assembly this route ran first), so - # the hidden-field closure is EMPTY rather than borrowed from another topic's contract. - import core.table_store as table_store - - kw.setdefault("table", - table_store.make(f"{scope_key}_table_workspace", st=session.runtime)) - hidden = frozenset() + elif scope_key.startswith("ut_"): + # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is + # `user_tables.may_open`, already applied by the assembly this route ran first), so + # the hidden-field closure is EMPTY rather than borrowed from another topic's contract. + import core.table_store as table_store + + kw.setdefault("table", + table_store.make(f"{scope_key}_table_workspace", st=session.runtime)) + hidden = frozenset() else: module, canonical = MODULE, aios_grid.FIELDS import core.table_store as table_store kw.setdefault("table", table_store.make( "customer_table_workspace", st=session.runtime)) hidden = perm_scope.hidden_keys(session.user, module, canonical, st=session.runtime) - return grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin, - # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the - # component's event objects verbatim, so a hidden key would otherwise arrive here with - # nothing between it and the store. - hidden_keys=hidden, - # ⭐ WAVE 25 (R6b, closes D-16) — THE TENANT HANDLE, on EVERY scope. This is the line - # D-16's exit condition names, and without it the rest of the fix is inert: the handler - # falls back to the module-global `core.store`, which is tenant #0's repo, so a Nurilab - # user's documents, cohorts and — through `_tops` — their whole customer/product table - # workspace were written into Royal Imports' dataset. Note it is set for the CUSTOMER - # and PRODUCT branches too, not only `ut_`: those two never passed a scoped `table`, so - # they were the ones actually resolving to tenant #0 on every request. - st=session.runtime, - fallback_ws=None, seen_ids={}, **kw) - - -#: The surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather -#: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with -#: `cohort_mode=True, scope_key='cohort'`. `product` (wave 16 C-TOPIC) is the SKU table: -#: same engine, its own pool, field contract and workspace BUCKET (routes_products). -_SCOPES = ("customer", "cohort", "product") - - -def _scope_or_400(raw): - """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would - hand a user the whole book on a page they opened to see one cohort — the widening direction, - which is the one that must fail closed. - - Wave 18 (C3-UT): a `ut_`-prefixed scope names a USER TABLE and passes through here — - existence and the per-table wall are enforced by `routes_tables.ut_assembly` (404/403), - which every consumer of such a scope goes through. Passing an unknown ut key therefore - still fails closed, just one layer down where the store can actually be consulted.""" - scope = (raw or "customer").strip().lower() - if scope.startswith("ut_"): - return scope - if scope not in _SCOPES: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(_SCOPES)} — refusing to guess") - return scope - - -@router.get("/workspace") -def workspace(scope: str = "customer", - session: Session = Depends(require_session)): - """The durable table workspace for this session: views, fields, overlays, folders. - - ⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope - asserts its own gate below. The old `module_gate("customer_data")` dependency 403'd a - `ut_*` workspace for any tenant whose catalogue omits the customer module (loopable/ - nurilab/gtmlab ship modules w/o it) — and the client then fell back to the shared - localStorage bucket + demo data, which is exactly the "new database shows RI's customer - fields" defect. A user table's real wall is `user_tables.may_open`, enforced inside - `ut_assembly` (404/403, one layer down where the store can be consulted). - - ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes - `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot - ride along in it without forking the thing that gate keeps single-sourced. Without a - workspace route the standalone shell's write path would be WRITE-ONLY: events persist - server-side and nothing reads them back on reload, so a saved view looks lost to the user - even though the store has it. This route closes that read-back gap at its own URL. - - `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the - wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a - full-access user. - """ - from core import grid_events - from routes_customers import grid_assembly - - scope = _scope_or_400(scope) - # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need - # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a - # session (its wall is the table's own, inside the assembly). - if not (scope == "product" or scope.startswith("ut_")): - session.require(MODULE) - # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The - # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the - # table's own wall (creator or admin, `user_tables.may_open`). - if scope.startswith("ut_"): - from routes_tables import ut_assembly - - storage_key = f"{session.tenant}:{scope}:{session.uname}" - try: - # ⭐⭐ W31-T20 / D-174 — `with_rows=False`. THIS ROUTE RENDERS NO ROW: the grid fetches - # them from `/tables/{key}/rows` or `/odoo-tables/{key}/rows` in the same paint, and - # `useCustomerData.load` fires both calls in one `Promise.all`. Building the table here - # bought nothing and cost everything — it is why `?scope=ut_odoo_gl_lines` answered - # `409 window_required` six times out of six on the live deploy while the ROWS door for - # the same table served a page in ~150 ms, i.e. a grid that could not be opened at all. - g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", "the tenant store is unavailable") - workspace = g["workspace"] - workspace["overlays"] = g["ws"].get("overlays") or {} - # The field contract rides the cheap re-read on EVERY topic — see the customer branch - # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)` - # is scope-parameterized, so `ut_` surfaces have their own sets). - workspace["fields"] = g["fields"] - workspace["measures"] = g["measures"] - workspace["measureSets"] = g["measure_sets"] - # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and - # `ut_assembly` has been building this topic's cohort CELLS since — but this route threw - # them away, so the Locked-views column arrived on the cheap re-read with permanently - # empty values. A column that exists and can never have one is worse than an absent - # column: it reads as "this record is in no locked view", which is a claim. - workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} - workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} - try: - from core import users as _users + return grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin, + # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the + # component's event objects verbatim, so a hidden key would otherwise arrive here with + # nothing between it and the store. + hidden_keys=hidden, + # ⭐ WAVE 25 (R6b, closes D-16) — THE TENANT HANDLE, on EVERY scope. This is the line + # D-16's exit condition names, and without it the rest of the fix is inert: the handler + # falls back to the module-global `core.store`, which is tenant #0's repo, so a Nurilab + # user's documents, cohorts and — through `_tops` — their whole customer/product table + # workspace were written into Royal Imports' dataset. Note it is set for the CUSTOMER + # and PRODUCT branches too, not only `ut_`: those two never passed a scoped `table`, so + # they were the ones actually resolving to tenant #0 on every request. + st=session.runtime, + fallback_ws=None, seen_ids={}, **kw) + + +#: The surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather +#: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with +#: `cohort_mode=True, scope_key='cohort'`. `product` (wave 16 C-TOPIC) is the SKU table: +#: same engine, its own pool, field contract and workspace BUCKET (routes_products). +_SCOPES = ("customer", "cohort", "product") + + +def _scope_or_400(raw): + """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would + hand a user the whole book on a page they opened to see one cohort — the widening direction, + which is the one that must fail closed. + + Wave 18 (C3-UT): a `ut_`-prefixed scope names a USER TABLE and passes through here — + existence and the per-table wall are enforced by `routes_tables.ut_assembly` (404/403), + which every consumer of such a scope goes through. Passing an unknown ut key therefore + still fails closed, just one layer down where the store can actually be consulted.""" + scope = (raw or "customer").strip().lower() + if scope.startswith("ut_"): + return scope + if scope not in _SCOPES: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(_SCOPES)} — refusing to guess") + return scope + + +@router.get("/workspace") +def workspace(scope: str = "customer", + session: Session = Depends(require_session)): + """The durable table workspace for this session: views, fields, overlays, folders. + + ⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope + asserts its own gate below. The old `module_gate("customer_data")` dependency 403'd a + `ut_*` workspace for any tenant whose catalogue omits the customer module (loopable/ + nurilab/gtmlab ship modules w/o it) — and the client then fell back to the shared + localStorage bucket + demo data, which is exactly the "new database shows RI's customer + fields" defect. A user table's real wall is `user_tables.may_open`, enforced inside + `ut_assembly` (404/403, one layer down where the store can be consulted). + + ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes + `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot + ride along in it without forking the thing that gate keeps single-sourced. Without a + workspace route the standalone shell's write path would be WRITE-ONLY: events persist + server-side and nothing reads them back on reload, so a saved view looks lost to the user + even though the store has it. This route closes that read-back gap at its own URL. + + `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the + wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a + full-access user. + """ + from core import grid_events + from routes_customers import grid_assembly + + scope = _scope_or_400(scope) + # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need + # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a + # session (its wall is the table's own, inside the assembly). + if not (scope == "product" or scope.startswith("ut_")): + session.require(MODULE) + # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The + # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the + # table's own wall (creator or admin, `user_tables.may_open`). + if scope.startswith("ut_"): + from routes_tables import ut_assembly + + storage_key = f"{session.tenant}:{scope}:{session.uname}" + try: + # ⭐⭐ W31-T20 / D-174 — `with_rows=False`. THIS ROUTE RENDERS NO ROW: the grid fetches + # them from `/tables/{key}/rows` or `/odoo-tables/{key}/rows` in the same paint, and + # `useCustomerData.load` fires both calls in one `Promise.all`. Building the table here + # bought nothing and cost everything — it is why `?scope=ut_odoo_gl_lines` answered + # `409 window_required` six times out of six on the live deploy while the ROWS door for + # the same table served a page in ~150 ms, i.e. a grid that could not be opened at all. + g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + workspace = g["workspace"] + workspace["overlays"] = g["ws"].get("overlays") or {} + # The field contract rides the cheap re-read on EVERY topic — see the customer branch + # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)` + # is scope-parameterized, so `ut_` surfaces have their own sets). + workspace["fields"] = g["fields"] + workspace["measures"] = g["measures"] + workspace["measureSets"] = g["measure_sets"] + # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and + # `ut_assembly` has been building this topic's cohort CELLS since — but this route threw + # them away, so the Locked-views column arrived on the cheap re-read with permanently + # empty values. A column that exists and can never have one is worse than an absent + # column: it reads as "this record is in no locked view", which is a claim. + workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} + workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} + try: + from core import users as _users workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) workspace["permissionUserOptions"] = _users.assignable_identities(tenant=session.tenant) - workspace["userAvatars"] = {k: v for k, v in - _users.avatar_map(tenant=session.tenant).items() - if str(v).startswith("data:image/")} + workspace["userAvatars"] = {k: v for k, v in + _users.avatar_map(tenant=session.tenant).items() + if str(v).startswith("data:image/")} except Exception: workspace["userOptions"] = [] workspace["permissionUserOptions"] = [] workspace["userAvatars"] = {} - workspace["scopeKey"] = scope - # ⭐ W31-T20 — R6's SECOND SENTENCE REACHES THE BROWSER. On a read-through grid too large - # for one window the pid set is empty, so cohort membership and a shared view's - # `memberPids` are UNRESOLVED here (both fail closed, and `workspace_wire` already stamps - # `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6 - # forbids; this is the announcement, and W31-T22 is the ticket that renders it. - workspace["limits"] = g.get("limits") or [] - return {"workspace": workspace} - - # ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field - # contract, own workspace BUCKET) and the contract's own storage key. It branches before - # the customer derivation because the customer storage key embeds (bu, agent) while the - # product one is deliberately `all:all` — one product workspace per user (the wave doc's - # C-TOPIC line), since BU narrows the product VALUES, not which workspace you own. - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, product_assembly - - # ⛔ THE GATE CHANGES WITH THE SCOPE (wave 21 C4: the dependency is session-only now, - # so this line IS the product wall — not a second one on top of customer_data). - session.require(PRODUCT_MODULE) - storage_key = f"{session.tenant}:product-list:{session.uname}:all:all" - try: - g = product_assembly(session, scope=scope, storage_key=storage_key) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", "the tenant store is unavailable") - workspace = g["workspace"] - workspace["overlays"] = g["ws"].get("overlays") or {} - # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own - # first-cohort contract change. Same reason as the customer branch below. - workspace["fields"] = g["fields"] - # The measure channel is EMPTY on this topic (customer-grain descope) — stated - # explicitly so the client's pickers grey rather than guess. - workspace["measures"] = g["measures"] - workspace["measureSets"] = g["measure_sets"] - # `g["derived"]` — the same correction as the user-table branch above, for the same - # reason: `product_assembly` builds this topic's cohort cells and this route discarded - # them. The measure half stays empty on this topic by descope, which is a different - # statement and one the offer already makes. - workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} - workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} - try: - from core import users as _users + workspace["scopeKey"] = scope + # ⭐ W31-T20 — R6's SECOND SENTENCE REACHES THE BROWSER. On a read-through grid too large + # for one window the pid set is empty, so cohort membership and a shared view's + # `memberPids` are UNRESOLVED here (both fail closed, and `workspace_wire` already stamps + # `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6 + # forbids; this is the announcement, and W31-T22 is the ticket that renders it. + workspace["limits"] = g.get("limits") or [] + return {"workspace": workspace} + + # ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field + # contract, own workspace BUCKET) and the contract's own storage key. It branches before + # the customer derivation because the customer storage key embeds (bu, agent) while the + # product one is deliberately `all:all` — one product workspace per user (the wave doc's + # C-TOPIC line), since BU narrows the product VALUES, not which workspace you own. + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, product_assembly + + # ⛔ THE GATE CHANGES WITH THE SCOPE (wave 21 C4: the dependency is session-only now, + # so this line IS the product wall — not a second one on top of customer_data). + session.require(PRODUCT_MODULE) + storage_key = f"{session.tenant}:product-list:{session.uname}:all:all" + try: + g = product_assembly(session, scope=scope, storage_key=storage_key) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + workspace = g["workspace"] + workspace["overlays"] = g["ws"].get("overlays") or {} + # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own + # first-cohort contract change. Same reason as the customer branch below. + workspace["fields"] = g["fields"] + # The measure channel is EMPTY on this topic (customer-grain descope) — stated + # explicitly so the client's pickers grey rather than guess. + workspace["measures"] = g["measures"] + workspace["measureSets"] = g["measure_sets"] + # `g["derived"]` — the same correction as the user-table branch above, for the same + # reason: `product_assembly` builds this topic's cohort cells and this route discarded + # them. The measure half stays empty on this topic by descope, which is a different + # statement and one the offer already makes. + workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} + workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} + try: + from core import users as _users workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) workspace["permissionUserOptions"] = _users.assignable_identities(tenant=session.tenant) - workspace["userAvatars"] = {k: v for k, v in - _users.avatar_map(tenant=session.tenant).items() - if str(v).startswith("data:image/")} + workspace["userAvatars"] = {k: v for k, v in + _users.avatar_map(tenant=session.tenant).items() + if str(v).startswith("data:image/")} except Exception: workspace["userOptions"] = [] workspace["permissionUserOptions"] = [] workspace["userAvatars"] = {} - workspace["scopeKey"] = scope - return {"workspace": workspace} - - # The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid` - # call sites) with the session's TENANT in the host's hardcoded slot, so localStorage state - # carries across embed ⇄ standalone on the same browser. - # Wave 15 R1 — the SAME derivation the pool uses (`routes_customers._team_agent`), so the - # storage key cannot drift from the scope it names. Value-identical for every migrated - # record (verify_perm_scope section D proves the derivation reproduces the legacy scope - # exactly), so nobody's saved localStorage state moves when the migration runs. - import core.perm_scope as perm_scope - team_id, agent = perm_scope.derive_pool_scope(session.user, MODULE) - bu = team_id if team_id is not None else "all" - if scope == "cohort": - storage_key = f"{session.tenant}:cohort:{session.uname}:{bu}" - else: - storage_key = f"{session.tenant}:customer-list:{session.uname}:{bu}:{agent or 'all'}" - - # ⛔ THE WIRE SHAPE IS THE SHARED PROJECTION (`aios_grid.workspace_wire`), not the store - # shape. The first version returned the store dict with no `storageKey` — and the client - # validator requires one, so the standalone shell DISCARDED the whole workspace: saved views - # never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer - # surface. One projection for both servers is the fix that cannot drift. - try: - g = grid_assembly(session, scope=scope, storage_key=storage_key) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", "the tenant store is unavailable") - workspace = g["workspace"] - # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the - # /customers rows, but this route is the cheap read-back a write can be verified against - # (verify_api's overlay probe) without paying the pool call. Per-user by construction — - # `table_workspace` is this session's workspace. - workspace["overlays"] = g["ws"].get("overlays") or {} - # ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration. - # - # `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked - # views" column ONLY when the caller owns at least one cohort. So a user's FIRST cohort - # CHANGES THE FIELD CONTRACT — and the rows call that used to be the only carrier of - # `fields` is deliberately never re-fetched on a write (it is a 15-minute-cached Odoo - # pull; this route is the cheap re-read). The client therefore had no way to learn about - # that column short of a remount, which is the second half of the owner's "it only shows - # up when I switch modules and come back". - # - # ⚠ `g["fields"]` is the SAME list `_payload` serves on `/customers` — same assembly, - # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree - # about what a column is. Taking it from anywhere else would fork the contract that - # verify_fields_contract.py exists to keep single-sourced. - workspace["fields"] = g["fields"] - # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ──────────────────────────── - # The embed receives these as top-level render args; standalone lifts them off THIS route - # (useCustomerData merges them into the payload slots the grid already reads). They ride - # the workspace rather than /customers because a durable write re-reads exactly this route - # (WORKSPACE_STALE), so a new measure column populates without refetching the heavy pool. - workspace["measures"] = g["measures"] - workspace["measureSets"] = g["measure_sets"] - # Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are - # strings; the client indexes with String(pid). - workspace["derived"] = {str(pid): cells for pid, cells in g["derived"].items()} - # Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns — - # the two other host-only render args the shell was missing (fail-closed without them: - # restricted fields uneditable, assignee picker empty). - workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} - try: - from core import users as _users + workspace["scopeKey"] = scope + return {"workspace": workspace} + + # The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid` + # call sites) with the session's TENANT in the host's hardcoded slot, so localStorage state + # carries across embed ⇄ standalone on the same browser. + # Wave 15 R1 — the SAME derivation the pool uses (`routes_customers._team_agent`), so the + # storage key cannot drift from the scope it names. Value-identical for every migrated + # record (verify_perm_scope section D proves the derivation reproduces the legacy scope + # exactly), so nobody's saved localStorage state moves when the migration runs. + import core.perm_scope as perm_scope + team_id, agent = perm_scope.derive_pool_scope(session.user, MODULE) + bu = team_id if team_id is not None else "all" + if scope == "cohort": + storage_key = f"{session.tenant}:cohort:{session.uname}:{bu}" + else: + storage_key = f"{session.tenant}:customer-list:{session.uname}:{bu}:{agent or 'all'}" + + # ⛔ THE WIRE SHAPE IS THE SHARED PROJECTION (`aios_grid.workspace_wire`), not the store + # shape. The first version returned the store dict with no `storageKey` — and the client + # validator requires one, so the standalone shell DISCARDED the whole workspace: saved views + # never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer + # surface. One projection for both servers is the fix that cannot drift. + try: + g = grid_assembly(session, scope=scope, storage_key=storage_key) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + workspace = g["workspace"] + # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the + # /customers rows, but this route is the cheap read-back a write can be verified against + # (verify_api's overlay probe) without paying the pool call. Per-user by construction — + # `table_workspace` is this session's workspace. + workspace["overlays"] = g["ws"].get("overlays") or {} + # ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration. + # + # `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked + # views" column ONLY when the caller owns at least one cohort. So a user's FIRST cohort + # CHANGES THE FIELD CONTRACT — and the rows call that used to be the only carrier of + # `fields` is deliberately never re-fetched on a write (it is a 15-minute-cached Odoo + # pull; this route is the cheap re-read). The client therefore had no way to learn about + # that column short of a remount, which is the second half of the owner's "it only shows + # up when I switch modules and come back". + # + # ⚠ `g["fields"]` is the SAME list `_payload` serves on `/customers` — same assembly, + # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree + # about what a column is. Taking it from anywhere else would fork the contract that + # verify_fields_contract.py exists to keep single-sourced. + workspace["fields"] = g["fields"] + # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ──────────────────────────── + # The embed receives these as top-level render args; standalone lifts them off THIS route + # (useCustomerData merges them into the payload slots the grid already reads). They ride + # the workspace rather than /customers because a durable write re-reads exactly this route + # (WORKSPACE_STALE), so a new measure column populates without refetching the heavy pool. + workspace["measures"] = g["measures"] + workspace["measureSets"] = g["measure_sets"] + # Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are + # strings; the client indexes with String(pid). + workspace["derived"] = {str(pid): cells for pid, cells in g["derived"].items()} + # Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns — + # the two other host-only render args the shell was missing (fail-closed without them: + # restricted fields uneditable, assignee picker empty). + workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} + try: + from core import users as _users workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) workspace["permissionUserOptions"] = _users.assignable_identities(tenant=session.tenant) - # Wave 14 C-AVATAR — the options vocabulary's companion: display name -> data URL. - # Absent entries fall back to the client's initials disc; a non-data: value is - # dropped (defence in depth beside the write-side wall in routes_auth). - workspace["userAvatars"] = {k: v for k, v in - _users.avatar_map(tenant=session.tenant).items() - if str(v).startswith("data:image/")} + # Wave 14 C-AVATAR — the options vocabulary's companion: display name -> data URL. + # Absent entries fall back to the client's initials disc; a non-data: value is + # dropped (defence in depth beside the write-side wall in routes_auth). + workspace["userAvatars"] = {k: v for k, v in + _users.avatar_map(tenant=session.tenant).items() + if str(v).startswith("data:image/")} except Exception: workspace["userOptions"] = [] workspace["permissionUserOptions"] = [] workspace["userAvatars"] = {} - - # THE SURFACE STAMP — a MIRROR of the host's own (`_table_grid`'s hide/cohort stamps). - # ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort - # list, and two switches for one behaviour would drift. - workspace["scopeKey"] = scope - if scope == "cohort": - workspace["cohortMode"] = True - # The create pane offers "this cohort table only" (default) vs "all customer tables". - workspace["scopeChoice"] = True - return {"workspace": workspace} - - -# ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02) -_TS_BUCKETS = ("week", "month", "quarter", "year") -_TS_MAX_BUCKETS = 120 -_TS_MAX_FIELDS = 12 -_TS_MAX_PIDS = 5000 -_TS_MAX_LAST_N = 120 -#: C-TSWIN (wave 14): sliding windows cost ONE aggregate query per (metric, bucket) — the -#: price of "bucket N == the grid cell at bucket N's end". The product cap keeps a legal -#: 120-bucket × 12-metric ask from becoming 1,440 queries in one request; typical panels -#: (12 × 3) sit two orders of magnitude under it. Dated amendment in the split doc. -_TS_MAX_CELLS = 720 - - -def _ts_start_of(bucket, d): - """The calendar START of the bucket holding `d` (week = Monday, the vocabulary rule).""" - if bucket == "week": - return d - dt.timedelta(days=d.weekday()) - if bucket == "month": - return d.replace(day=1) - if bucket == "quarter": - return d.replace(month=((d.month - 1) // 3) * 3 + 1, day=1) - return d.replace(month=1, day=1) - - -def _ts_next(bucket, d): - if bucket == "week": - return d + dt.timedelta(days=7) - if bucket == "year": - return d.replace(year=d.year + 1) - step = 1 if bucket == "month" else 3 - m = d.month + step - return dt.date(d.year + (m - 1) // 12, (m - 1) % 12 + 1, 1) - - -def _ts_prev(bucket, d): - if bucket == "week": - return d - dt.timedelta(days=7) - if bucket == "year": - return d.replace(year=d.year - 1) - step = 1 if bucket == "month" else 3 - y, m = d.year, d.month - step - while m < 1: - m += 12 - y -= 1 - return dt.date(y, m, 1) - - -def _ts_label(bucket, start): - if bucket == "week": - return f"Wk of {start.strftime('%b %d')}" - if bucket == "month": - return start.strftime("%b %Y") - if bucket == "quarter": - return f"Q{(start.month - 1) // 3 + 1} {start.year}" - return str(start.year) - - -@router.post("/grid/timeseries") -def grid_timeseries(body: dict = Body(default=None), scope: str = "customer", - session: Session = Depends(module_gate(MODULE))): - """Pooled measure values per calendar bucket — the time-series view's data channel. - - C-TSWIN (wave 14, ruling R1): AS-OF semantics. Each metric's OWN stored window is - re-resolved per bucket with `today := min(bucket end, real today)` — `ytd` is cumulative - from Jan 1, `last_90_days` trailing, `all_time` cumulative ever; bucket N equals what the - grid's measure column would show if today were bucket N's end. Fixed-range (`custom`) - windows cannot slide and are dropped per field as `window_fixed`. Rows carry - `window: {kind, label}` so a cumulative row cannot be misread as periodic, and there is - NO `total` column — sliding windows overlap, so a sum of columns would double-count. - - The client sends the pids its view currently matches (the filter IS the scope); the server - intersects them with the session's book, so the request can only ever NARROW. Values are - POOLED aggregates computed by the semantic layer's own expression (the additivity law: an - average is computed at pool grain, never averaged over per-customer answers). A bucket - with no rows is 0 for a sum/count and null for anything else; a bucket that has not - STARTED yet is null for every kind — an unstarted period's YTD is unanswered, not 0. - """ - from core import measure_resolve - from harness import windows as _wn - from routes_customers import _pool_stamp, _team_agent, allowed_pids - - # Wave 16 C-TOPIC: the channel is CUSTOMER-GRAIN (measure_resolve's own grain), so the - # product surface is refused in words rather than answered with an id-space accident — - # product pids are CRC32 hashes and would mostly fall outside the customer book anyway, - # but "mostly" is not a wall. The client hides the mode for this topic; this is the - # server's half of the same refusal. - _sc = _scope_or_400(scope) - if _sc == "product" or _sc.startswith("ut_"): - raise err(400, "bad_timeseries", - "this surface has no time-series channel — measures are customer-grain") - - body = body or {} - bucket = body.get("bucket") - if bucket not in _TS_BUCKETS: - raise err(400, "bad_timeseries", "bucket must be one of week, month, quarter, year") - raw_fields = body.get("fields") - if not isinstance(raw_fields, list) or not raw_fields: - raise err(400, "bad_timeseries", "fields must be a non-empty list of field keys") - if len(raw_fields) > _TS_MAX_FIELDS: - raise err(400, "bad_timeseries", f"at most {_TS_MAX_FIELDS} fields per request") - raw_pids = body.get("pids") - if not isinstance(raw_pids, list) or not raw_pids: - raise err(400, "bad_timeseries", "pids must be a non-empty list") - if len(raw_pids) > _TS_MAX_PIDS: - raise err(400, "bad_timeseries", f"at most {_TS_MAX_PIDS} pids per request") - try: - wanted = {int(p) for p in raw_pids} - except (TypeError, ValueError): - raise err(400, "bad_timeseries", "pids must be integers") - pool = wanted & {int(p) for p in allowed_pids(session)} - if not pool: - # 403, not an empty 200: the caller asked about customers outside their book, and an - # all-zero series over nobody would read as "no activity", not "not yours". - raise err(403, "out_of_scope", "none of those customers are in your book") - - span = body.get("span") - if not isinstance(span, dict): - raise err(400, "bad_timeseries", "span must be {'lastN': n} or {'from': .., 'to': ..}") - today = time.strftime("%Y-%m-%d") - t = dt.date.fromisoformat(today) - n = span.get("lastN") - starts = [] - if n is not None: - if not isinstance(n, int) or isinstance(n, bool) or not (1 <= n <= _TS_MAX_LAST_N): - raise err(400, "bad_timeseries", f"lastN must be 1..{_TS_MAX_LAST_N}") - cur = _ts_start_of(bucket, t) - starts = [cur] - for _ in range(n - 1): - cur = _ts_prev(bucket, cur) - starts.append(cur) - starts.reverse() - else: - try: - d_from = dt.date.fromisoformat(str(span.get("from"))) - d_to = dt.date.fromisoformat(str(span.get("to"))) - except (TypeError, ValueError): - raise err(400, "bad_timeseries", "span.from/to must be ISO dates (YYYY-MM-DD)") - if d_from > d_to: - d_from, d_to = d_to, d_from - cur = _ts_start_of(bucket, d_from) - while cur <= d_to: - starts.append(cur) - if len(starts) > _TS_MAX_BUCKETS: - raise err(400, "bad_timeseries", - f"that span is more than {_TS_MAX_BUCKETS} {bucket} buckets - " - f"narrow it") - cur = _ts_next(bucket, cur) - if not starts: - raise err(400, "bad_timeseries", "the span holds no buckets") - ends = [_ts_next(bucket, s) - dt.timedelta(days=1) for s in starts] - - rt = session.runtime - if not rt.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - import modules.customer_data as cl_mod - # Read-only route: it must not consume a one-shot label-correction ack the browser has - # not seen yet (the same rule event validation follows). - ws = cl_mod.table_workspace(session.uname, consume_corrections=False) - fdefs = ws.get("fields") or {} - keys, dropped, seen = [], [], set() - for k in raw_fields: - k = str(k or "") - if not k or k in seen: - continue - seen.add(k) - fd = fdefs.get(k) - if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): - # Named, never silent: v1 eligibility is measure-backed fields only (C-TS). - dropped.append({"field": k, "reason": "not_a_measure_field"}) - continue - keys.append(k) - # ⚡ WAVE 17 R9 (amendment 2026-08-03, GRID's cross-fence ask) — A SHEET WITH NO MEASURE - # FIELDS IS NO LONGER A 400. It answers with the BUCKET GRID and no rows. - # - # Why this is the honest direction and not a loosening: the bucket starts and their labels - # are SERVER math (`_ts_start_of` / `_ts_next` / `_ts_label`), and R9 has the client - # synthesizing snapshot rows for preset columns that have no window to slide. Those rows - # must be painted under the SAME headings as everything else, so the client needs the - # columns even when the server has no series to put in them. Refusing the whole request - # meant the panel could offer nothing at all on this tenant — the shipped contract has zero - # measure-backed presets (`aios_grid.py`: "this branch is currently MEMBERLESS"), which is - # what item 5 is actually about. - # - # Nothing is invented by this: `rows` is empty and `meta.dropped` NAMES every field and why. - # A 400 is still returned for a request that is malformed (bad bucket, bad span, no fields - # at all) — this only stops treating "I asked about columns you cannot serve" as an error. - mfields, slid_keys = [], [] - for k in keys: - m = dict(fdefs[k]["measure"]) - if (m.get("window") or {}).get("kind") == "custom": - # C-TSWIN: a fixed date range cannot slide across buckets — named, never silent, - # the same posture as not_a_measure_field. - dropped.append({"field": k, "reason": "window_fixed"}) - continue - mfields.append({"key": k, "measure": m}) - slid_keys.append(k) - # Same amendment as above: every metric being fixed-range is a sheet with no SERIES, not a - # broken request. The columns still stand, and `window_fixed` still names each refusal. - if len(starts) * len(mfields) > _TS_MAX_CELLS: - raise err(400, "bad_timeseries", - "that ask is too wide - narrow the span or pick fewer metrics") - - team_id, agent = _team_agent(session) - stamp = _pool_stamp(rt, team_id, agent) - problems = [] - bucket_bounds = [(s.isoformat(), e.isoformat()) for s, e in zip(starts, ends)] - # No slidable metrics = nothing to resolve. Skipping the call rather than asking the - # resolver about an empty list keeps the memo free of a meaningless key. - answers = measure_resolve.series_values( - mfields, bucket, bucket_bounds, - team_id, frozenset(pool), today, stamp, rt.series_memo, - on_error=lambda tag, e: problems.append(str(e)[:200])) if mfields else {} - - columns = [] - for s, e in zip(starts, ends): - col = {"key": s.isoformat(), "label": _ts_label(bucket, s), - "from": s.isoformat(), "to": e.isoformat()} - if e > t: - col["partial"] = True - columns.append(col) - rows = [] - for k in slid_keys: - ans = answers.get(k) - if ans is None: - dropped.append({"field": k, "reason": "unresolvable"}) - continue - vals_by = ans.get("values") or {} - agg_kind = str(ans.get("agg") or "sum") - zero_fill = agg_kind in ("sum", "count") - vals = [] - for s in starts: - if s > t: - # C-TSWIN: an unstarted period is unanswered for EVERY agg kind — a future - # month's YTD zero-filled to 0 would read as "the year reset". - vals.append(None) - else: - vals.append(vals_by.get(s.isoformat(), 0 if zero_fill else None)) - wspec = (fdefs[k].get("measure") or {}).get("window") - wnorm = _wn.normalize(wspec) or {} - rows.append({"field": k, "label": str(fdefs[k].get("label") or k)[:120], - "agg": agg_kind, "values": vals, - "window": {"kind": str(wnorm.get("kind") or ""), - "label": _wn.label(wspec)}}) - meta = {"pool": len(pool), "today": today, "bucket": bucket} - if dropped: - meta["dropped"] = dropped - if problems: - meta["problems"] = problems[:5] - return {"columns": columns, "rows": rows, "meta": meta} - - -#: C-CAL caps (wave 17, owner item 9 / ruling R4). A month of days, a handful of metrics. -#: Every one of these is a 400 WITH ITS REASON, never a silent trim: a calendar that quietly -#: answered 20 of the 31 days asked about would paint eleven blank cells that look like days -#: with no activity. -_CAL_MAX_GROUPS = 31 -_CAL_MAX_FIELDS = 6 -_CAL_MAX_CELLS = 186 - - -@router.post("/grid/calendar_metrics") -def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "customer", - session: Session = Depends(module_gate(MODULE))): - """Per-DAY measure values with each metric's own window slid to that day (C-CAL / R4). - - ⛔ WHY THIS EXISTS AT ALL. The calendar's summary cells used to aggregate the row VALUES the - grid already held — which for a measure column means "every member's YTD **as of today**", - summed and printed under a date in March. The number was arithmetically fine and semantically - a lie: it answered a question about today while sitting in a cell labelled with another day. - R4: a metric in a day cell is computed AS OF THAT DAY. - - The shape differs from the time-series channel in exactly one way, and it is the reason this - is a separate route rather than a parameter: **every group carries its OWN pid set**. A - calendar day holds the records the date field placed there, so day-to-day the subject - changes. One `allowed_pids` for the whole request — the TS channel's shape — would compute - each day over everybody, which is a different question again. - - Static (non-measure) fields are NOT served here. They have no window to slide, so the - client's own per-day aggregation over row values stays correct for them; sending them would - invite a second implementation of arithmetic that already works. - """ - from core import measure_resolve - from harness import windows as _wn - from routes_customers import _pool_stamp, _team_agent, allowed_pids - - # The same refusal the TS channel makes, for the same reason: measures are customer-grain - # and product pids are CRC32 hashes of SKU codes. "Mostly outside the book" is not a wall. - _sc_cal = _scope_or_400(scope) - if _sc_cal == "product" or _sc_cal.startswith("ut_"): - raise err(400, "bad_calendar_metrics", - "the product surface has no measure channel yet — measures are customer-grain") - - body = body or {} - raw_groups = body.get("groups") - if not isinstance(raw_groups, list) or not raw_groups: - raise err(400, "bad_calendar_metrics", - "groups must be a non-empty list of {key: 'YYYY-MM-DD', pids: [...]}") - if len(raw_groups) > _CAL_MAX_GROUPS: - raise err(400, "bad_calendar_metrics", - f"at most {_CAL_MAX_GROUPS} days per request (one month)") - raw_fields = body.get("fields") - if not isinstance(raw_fields, list) or not raw_fields: - raise err(400, "bad_calendar_metrics", "fields must be a non-empty list of field keys") - if len(raw_fields) > _CAL_MAX_FIELDS: - raise err(400, "bad_calendar_metrics", f"at most {_CAL_MAX_FIELDS} metrics per request") - if len(raw_groups) * len(raw_fields) > _CAL_MAX_CELLS: - raise err(400, "bad_calendar_metrics", - "that ask is too wide - fewer days or fewer metrics") - - book = {int(p) for p in allowed_pids(session)} - groups, seen_days = [], set() - for g in raw_groups: - if not isinstance(g, dict): - raise err(400, "bad_calendar_metrics", "every group must be an object") - day = str(g.get("key") or "") - try: - d = dt.date.fromisoformat(day) - except (TypeError, ValueError): - raise err(400, "bad_calendar_metrics", - "every group key must be an ISO date (YYYY-MM-DD)") - if day in seen_days: - raise err(400, "bad_calendar_metrics", f"day {day} appears twice") - seen_days.add(day) - raw_pids = g.get("pids") - if not isinstance(raw_pids, list): - raise err(400, "bad_calendar_metrics", "every group needs a pids list") - try: - wanted = {int(p) for p in raw_pids} - except (TypeError, ValueError): - raise err(400, "bad_calendar_metrics", "pids must be integers") - # NARROW-ONLY, per group. The client sends what its calendar placed; the server can - # only ever remove from that, never add. - groups.append((day, d, frozenset(wanted & book))) - if sum(len(p) for _, _, p in groups) > _TS_MAX_PIDS: - raise err(400, "bad_calendar_metrics", - f"at most {_TS_MAX_PIDS} customer references per request") - - rt = session.runtime - if not rt.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - import modules.customer_data as cl_mod - ws = cl_mod.table_workspace(session.uname, consume_corrections=False) - fdefs = ws.get("fields") or {} - keys, dropped, seen = [], [], set() - for k in raw_fields: - k = str(k or "") - if not k or k in seen: - continue - seen.add(k) - fd = fdefs.get(k) - if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): - # The TS channel's vocabulary, deliberately reused rather than re-coined: the client - # already knows how to say these three words to a reader. - dropped.append({"field": k, "reason": "not_a_measure_field"}) - continue - if ((fd["measure"].get("window") or {}).get("kind") == "custom"): - dropped.append({"field": k, "reason": "window_fixed"}) - continue - keys.append(k) - if not keys: - raise err(400, "bad_calendar_metrics", - "none of the requested fields are measure fields with a window that can " - "slide to a day") - - today = time.strftime("%Y-%m-%d") - t = dt.date.fromisoformat(today) - team_id, agent = _team_agent(session) - stamp = _pool_stamp(rt, team_id, agent) - problems = [] - values = {k: {} for k in keys} - for day, d, pids in groups: - # A day that has not happened is unanswered for EVERY aggregate kind — the unstarted - # bucket law at day grain. Answering 0 would say "we sold nothing", which is a claim - # about a day nobody has lived through yet. - if d > t or not pids: - for k in keys: - values[k][day] = None - continue - answers = measure_resolve.series_values( - [{"key": k, "measure": dict(fdefs[k]["measure"])} for k in keys], - "day", [(day, day)], team_id, pids, today, stamp, rt.series_memo, - on_error=lambda tag, e: problems.append(str(e)[:200])) - for k in keys: - ans = answers.get(k) - if ans is None: - values[k][day] = None - continue - vals_by = ans.get("values") or {} - zero_fill = str(ans.get("agg") or "sum") in ("sum", "count") - values[k][day] = vals_by.get(day, 0 if zero_fill else None) - - for k in keys: - if all(v is None for v in values[k].values()): - # Every day unanswerable is a FIELD-level failure, and saying so is the difference - # between "no activity that month" and "this metric could not be computed". - dropped.append({"field": k, "reason": "unresolvable"}) - out = {"values": values, "today": today, - "windows": {k: {"kind": str((_wn.normalize( - (fdefs[k].get("measure") or {}).get("window")) or {}).get("kind") or ""), - "label": _wn.label((fdefs[k].get("measure") or {}).get("window"))} - for k in keys}} - if dropped: - out["dropped"] = dropped - if problems: - out["problems"] = problems[:5] - return out - - -#: ⭐⭐ THE BULK CELL DOOR'S OWN CEILING, and it is REPORTED rather than silently enforced. The -#: standing no-cap rule governs data read from a connected source; this is a WRITE of team-typed -#: values, so a bound is right — but R6's second sentence still binds, which is why exceeding it -#: is a 400 naming the number and the count, never a truncation. -#: Sized against the real job with headroom: the 2027 catalog is 1,397 rows. -MAX_BULK_ROWS = 10_000 - - -@router.post("/grid/bulk-cells") -def bulk_cells(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """`{scopeKey, rows: {"": {key: value}}}` → shared cells on many rows, ONE transaction. - - ⭐⭐ WHY THIS EXISTS AT ALL, because "we already have `/grid/events`" is the obvious objection. - That door is capped at `_MAX_EVENTS = 24` (it is sized for the client's resend window), so a - 1,397-row import is ~59 sequential POSTs against ONE JSON document — and this repo has a - MEASURED scar for exactly that: 18 writes against one document under the store's coalescing - single-flight landed **zero** while answering 200 eighteen times. Raising `_MAX_EVENTS` would - have widened the browser's own resend window to fix a script's problem. One door, one - transaction, is the honest shape. - - ⛔ TENANT-WIDE KEYS ONLY, AND THAT IS THE POINT RATHER THAN A LIMITATION. It writes through - `shared_overlay`, which every account in the tenant reads. A per-user bulk write would be a - contradiction: nobody imports 1,397 rows of the team's work so that one account can see it — - that is precisely the failure owner ruling R6 was made to avoid. A key that is not declared - `shared: true` in the canonical contract is REFUSED BY NAME. - - ⛔ ADMIN ONLY. This mutates data every account in the tenant reads, in bulk, in one call. - D-172 already records that the shared stratum has an asymmetric wall (anyone may create a - tenant-wide column, only the creator or an admin may delete one); a new door does not get to - inherit the loose half of an asymmetry somebody already flagged. - """ - import core.shared_overlay as shared_overlay - import modules.product_data as pd - import core.perm_scope as perm_scope - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - scope = _scope_or_400((body or {}).get("scopeKey")) - if scope != "product": - # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. - raise err(400, "scope_not_bulk_writable", - f"bulk cell writes are available on the product database; '{scope}' has no " - f"tenant-wide cell stratum") - session.require(PRODUCT_MODULE) - if not session.admin: - raise err(403, "admin_only", - "a bulk write changes values every account in this workspace reads") - - rows = (body or {}).get("rows") - if not isinstance(rows, dict): - raise err(400, "bad_rows", "expected {rows: {\"\": {field: value}}}") - if len(rows) > MAX_BULK_ROWS: - raise err(400, "too_many_rows", - f"at most {MAX_BULK_ROWS} rows per request; this one carried {len(rows)}") - - pids, _team, _src, fields_base = scoped_pool(session) - allowed = {int(p) for p in pids} - shared_keys = set(pd.SHARED_KEYS()) - hidden = perm_scope.hidden_keys(session.user, PRODUCT_MODULE, fields_base) - - clean, unknown_pid, refused_keys = {}, [], {} - for raw_pid, values in rows.items(): - try: - pid = int(raw_pid) - except (TypeError, ValueError): - unknown_pid.append(str(raw_pid)[:40]) - continue - # ⚠ THE POOL IS THE WALL. `scoped_pool` is the same predicate the read door uses, so a - # caller cannot write a row they could not see — including a row in another BU. - if pid not in allowed: - unknown_pid.append(str(raw_pid)[:40]) - continue - keep = {} - for key, value in dict(values or {}).items(): - key = str(key) - if key not in shared_keys: - refused_keys.setdefault("not_shared", set()).add(key) - continue - if key in hidden: - refused_keys.setdefault("hidden_by_permissions", set()).add(key) - continue - if not isinstance(value, (str, int, float)) or isinstance(value, bool): - refused_keys.setdefault("unsupported_value", set()).add(key) - continue - keep[key] = value - if keep: - clean[pid] = keep - - written = shared_overlay.put_rows(pd.TABLE_KEY, clean, st=pd.TABLE_OPS.st) if clean else {} - out = { - "rows_written": len(written), - "cells_written": sum(len(v) for v in written.values()), - "rows_requested": len(rows), - } - # ⭐ R6's second sentence: what did NOT land, and why. A bulk door that reports only its - # successes is how 701 unmatched SKUs disappear quietly. - if unknown_pid: - out["rows_not_in_your_pool"] = {"count": len(unknown_pid), - "sample": sorted(unknown_pid)[:10]} - if refused_keys: - out["refused_fields"] = {reason: sorted(keys) - for reason, keys in refused_keys.items()} - return out - - -@router.post("/grid/events") -def grid_events_route(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """`{events: []}` → `{results, doc?, toast?}`. - - Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the - same sessions the read door (`/workspace`) admits, or a tenant without the customer module - can SEE its own user tables and not write to them.""" - from core import grid_events - from routes_customers import grid_assembly - - events = (body or {}).get("events") - if events is None and isinstance(body, dict) and body.get("type"): - events = [body] # a single event object, the legacy shape - if not isinstance(events, list): - raise err(400, "bad_events", "expected {events: [...]}") - if len(events) > _MAX_EVENTS: - raise err(400, "too_many_events", - f"at most {_MAX_EVENTS} events per request (the client's resend window)") - - # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be - # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a - # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on - # what a scope is, or the surface you read is not the surface you wrote. - scope = _scope_or_400((body or {}).get("scopeKey")) - # Wave 21 C4 — same per-scope gate as /workspace (read and write doors must agree). - if not (scope == "product" or scope.startswith("ut_")): - session.require(MODULE) - # This assembly validates the write; it is not a payload the browser will render. Leave - # one-shot field-name correction acks queued for the subsequent /workspace refresh. - # Wave 16 C-TOPIC: the PRODUCT topic gets the product assembly — product field contract, - # product pids, and (below) the product TABLE OPS, so a product event is validated against - # and lands in the product bucket. The measure/cohort context is honestly EMPTY there: - # `clean_measure_field` refuses measure creates on this surface by construction (the - # customer-grain descope), which is the fail-closed shape, not an accident. - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, product_assembly - - session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4) - g = product_assembly(session, consume_corrections=False) - elif scope.startswith("ut_"): - # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the - # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here). - # ⭐⭐ WAVE 30 / W30-T30 (owner item 2: *"when I click hide fields it crash… no matter the - # size"*). This used to be `ut_assembly(...)`, which builds the whole table to validate a - # write it then throws away: `scoped_pool` allocates a dict per row and sorts them, so ONE - # hide-fields checkbox rebuilt ~33k order rows before the event was even dispatched. - # ⛔ NOTHING IS VALIDATED LESS. The comment above still holds — this assembly is the - # permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys - # this route reads, with a pid set derived from exactly the row ids `scoped_pool` would - # have kept. What it does not do is materialise the rows nobody here looks at. - from routes_tables import ut_write_ctx - - g = ut_write_ctx(session, scope) - # ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the - # half that is easy to skip because the code already "fails closed" without it. When a - # read-through grid's population exceeds one window the pid set is EMPTY, and every - # pid-bearing handler in `grid_events` then answers `False` — `overlay_patch` and - # `add_to_list` both `return False` for a pid not in `allowed_pids`. On the wire that is - # `rerender: false` and HTTP 200: a write the user watched succeed, that did nothing - # ([[lost-write-looks-like-failed-read]]). Schema-only events — hide a field, save a view, - # rename a column — name no pid and are untouched, which is D-170. - # ⚠ THE TEST IS THE EVENT'S OWN KEYS, not a kind list: a new pid-bearing event type would - # otherwise inherit the silent no-op the day it is added. - if g.get("limits"): - named = [e for e in events - if isinstance(e, dict) and (e.get("pid") is not None or e.get("pids"))] - if named: - lim = g["limits"][0] - raise err(409, "pid_scope_unresolved", - f"this database is served through the connector mirror and its rows " - f"cannot be listed in one window, so a change addressed to particular " - f"records ({len(named)} of {len(events)} here) cannot be admitted — " - f"{lim.get('cause') or 'the row set is unresolved'}. " - f"{lim.get('recommendation') or ''}".strip()) - else: - g = grid_assembly(session, scope=scope, consume_corrections=False) - # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`, - # `clean_measure_field` had an empty admission list and every measure-column create over - # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every - # measure CONDITION out of a saved view. The embed always passed these; the API adapter - # simply had not been given them — the standalone shell could read measures it could - # never write. - ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, - measure_keys=frozenset(m["key"] for m in g["measures"]), - resolved_ids=frozenset(g["measure_sets"]), - cohort_ids=frozenset(c["id"] for c in g["lists"]), - measure_offer=tuple(g["measures"]), - visible_views=tuple(g["views"])) - - # Per-event results so the client can tell which of a batch landed — the component's own - # bridge has no response channel at all, so this is strictly more than the embed gets. - results = [] - try: - for one in events: - eid = str(one.get("id") or "") if isinstance(one, dict) else "" - # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler - # appends to a shared list, so the refusals belonging to THIS event are exactly the - # ones that appeared across THIS call. Reading the list once after the loop would - # answer "the batch was refused" and leave the caller to guess which member, which is - # the same class of unfalsifiable answer the channel exists to end. - _before = len(ctx.out.refusals) - rerender = grid_events.handle_one(one, ctx) - row = {"id": eid, "rerender": bool(rerender)} - mine = ctx.out.refusals[_before:] - if mine: - # `refused` is the SHAPE a caller branches on; the first reason is the one that - # stopped this write (a handler returns at its first refusal). - row["refused"] = mine[0] - results.append(row) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable — none of your changes were saved") - - out = {"results": results, "rerender": any(r["rerender"] for r in results)} - if ctx.out.refusals: - # ⚠ ALSO AT THE TOP LEVEL, because a batch that was wholly refused must not read as a - # batch that wholly landed. A caller that only checks the envelope still learns something - # is wrong, and a caller that walks `results` learns exactly which member. - out["refusals"] = list(ctx.out.refusals) - if ctx.out.doc is not None: - # ⭐ C4 / W30-T27 — `docPayload` IS THE NAME THE CLIENT ALREADY DECLARES. `types.ts` has - # carried `docPayload?: {pid, docId, name, mime, data_b64}` since C5, and `Documents.tsx` - # matches it against the fetch it is waiting on — while this route has been answering - # `doc`, which `apiBridge.ts` deliberately drops. One object, emitted under the name the - # consumer looks for, so F's wiring needs no translation step to get wrong. - # ⚠ `doc` stays for one wave: nothing in the client reads it, but a gate might, and - # removing a key to save six bytes is not worth a red nobody predicted. - out["doc"] = out["docPayload"] = ctx.out.doc - if ctx.out.toast is not None: - out["toast"] = ctx.out.toast - - # ── ⭐ owner item 2 (2026-08-03): THE NEW MEASURE COLUMN'S VALUES, ONE ROUND TRIP SOONER ── - # - # Creating a measure column cost the browser TWO sequential trips before a single number - # appeared: this one to persist the field, then a whole `/workspace` to compute it. The - # second cannot start until the first lands (the resolver reads the PERSISTED field), so the - # wait was structural, not slow code — the owner's "it takes some time for the data to - # populate". The values are computed here instead, immediately after the write, and ride - # this response. - # - # ⚠ IT COSTS NOTHING EXTRA TO COMPUTE. The expensive part is one DuckDB aggregate over the - # book, and `rt.measure_memo` is keyed on (pool stamp, scope, pool, measure, window) — so - # the `/workspace` re-read that still follows HITS the memo instead of doing this work. The - # query happens once either way; only its position moved. - # - # ⚠ NARROW ON PURPOSE. Gated to an actual measure-column write, so an overlay edit or a - # cohort add — the overwhelming majority of events — never pays for a second assembly. - # - # ⚠ AND IT IS A SHORTCUT, NOT A PATH. Any failure is swallowed: `WORKSPACE_STALE` still - # fires from `rerender`, and the re-read still delivers these values exactly as it does - # today. Nothing depends on this having worked. - if out["rerender"] and scope in ("customer", "cohort") and any( - isinstance(e, dict) and e.get("type") == "field_upsert" - and str(((e.get("field") or {}) if isinstance(e.get("field"), dict) else {}) - .get("key") or "").startswith("measure_") - for e in events): - try: - fresh = grid_assembly(session, scope=scope, consume_corrections=False) - out["derived"] = {str(pid): cells for pid, cells in fresh["derived"].items()} - except Exception: - pass - # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped - # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an - # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views, - # folders, cohorts) is re-read from the store on the next request. An earlier version cleared - # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh - # data that was never in it. - return out - - -# ── CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS ─────────────────────────────────── -# ⭐⭐ R6 — *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table` -# is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never -# imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid` -# behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as -# `routes_odoo_tables` declares connected tables to `user_tables.register_connected`. -# -# ⛔ REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files -# are outside wave 36's lane-C fence. The readers themselves are three lines each and call the -# SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no -# second scope derivation — only a second CALLER of the one that exists. -# -# ⚠ AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` + -# `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and -# `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done -# here, because neither file is in this fence. What ships now is the arm the wave is load-bearing -# on — every `ut_*` database, plus E's sandbox — and a topic arm that is REAL rather than stubbed, -# so `scoped_table`'s topic leg is exercised by the product instead of only by a gate. -def _topic_rt(st, module): - """The tenant runtime a topic pool must be built against, or a REPORTED refusal. - - ⛔ A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the - default" without picking a tenant at random — which on this box is tenant #0's PRODUCTION - data. Standing rule 1's second sentence: say why, and say what to do instead. - """ - if st is None: - import core.perm_scope as perm_scope - raise perm_scope.Unresolvable( - subject="rows", effect="unreadable", - cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant " - f"runtime was passed", - recommendation="pass the session's runtime as `st=`. A topic pool cannot be " - "resolved without knowing which tenant is asking") - return st - - -def _customer_rows(table_key, user, st): - """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" - import aios_grid - import core.perm_scope as perm_scope - from routes_customers import _pool_for - - rt = _topic_rt(st, table_key) - team_id, agent = perm_scope.derive_pool_scope(user, table_key) - return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) - - -def _product_rows(table_key, user, st): - """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a - BU-pinned reader gets that BU's field contract rather than the consolidated one.""" - import core.perm_scope as perm_scope - from routes_products import _pool_for, pd_fields - - rt = _topic_rt(st, table_key) - team_id, _agent = perm_scope.derive_pool_scope(user, table_key) + + # THE SURFACE STAMP — a MIRROR of the host's own (`_table_grid`'s hide/cohort stamps). + # ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort + # list, and two switches for one behaviour would drift. + workspace["scopeKey"] = scope + if scope == "cohort": + workspace["cohortMode"] = True + # The create pane offers "this cohort table only" (default) vs "all customer tables". + workspace["scopeChoice"] = True + return {"workspace": workspace} + + +# ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02) +_TS_BUCKETS = ("week", "month", "quarter", "year") +_TS_MAX_BUCKETS = 120 +_TS_MAX_FIELDS = 12 +_TS_MAX_PIDS = 5000 +_TS_MAX_LAST_N = 120 +#: C-TSWIN (wave 14): sliding windows cost ONE aggregate query per (metric, bucket) — the +#: price of "bucket N == the grid cell at bucket N's end". The product cap keeps a legal +#: 120-bucket × 12-metric ask from becoming 1,440 queries in one request; typical panels +#: (12 × 3) sit two orders of magnitude under it. Dated amendment in the split doc. +_TS_MAX_CELLS = 720 + + +def _ts_start_of(bucket, d): + """The calendar START of the bucket holding `d` (week = Monday, the vocabulary rule).""" + if bucket == "week": + return d - dt.timedelta(days=d.weekday()) + if bucket == "month": + return d.replace(day=1) + if bucket == "quarter": + return d.replace(month=((d.month - 1) // 3) * 3 + 1, day=1) + return d.replace(month=1, day=1) + + +def _ts_next(bucket, d): + if bucket == "week": + return d + dt.timedelta(days=7) + if bucket == "year": + return d.replace(year=d.year + 1) + step = 1 if bucket == "month" else 3 + m = d.month + step + return dt.date(d.year + (m - 1) // 12, (m - 1) % 12 + 1, 1) + + +def _ts_prev(bucket, d): + if bucket == "week": + return d - dt.timedelta(days=7) + if bucket == "year": + return d.replace(year=d.year - 1) + step = 1 if bucket == "month" else 3 + y, m = d.year, d.month - step + while m < 1: + m += 12 + y -= 1 + return dt.date(y, m, 1) + + +def _ts_label(bucket, start): + if bucket == "week": + return f"Wk of {start.strftime('%b %d')}" + if bucket == "month": + return start.strftime("%b %Y") + if bucket == "quarter": + return f"Q{(start.month - 1) // 3 + 1} {start.year}" + return str(start.year) + + +@router.post("/grid/timeseries") +def grid_timeseries(body: dict = Body(default=None), scope: str = "customer", + session: Session = Depends(module_gate(MODULE))): + """Pooled measure values per calendar bucket — the time-series view's data channel. + + C-TSWIN (wave 14, ruling R1): AS-OF semantics. Each metric's OWN stored window is + re-resolved per bucket with `today := min(bucket end, real today)` — `ytd` is cumulative + from Jan 1, `last_90_days` trailing, `all_time` cumulative ever; bucket N equals what the + grid's measure column would show if today were bucket N's end. Fixed-range (`custom`) + windows cannot slide and are dropped per field as `window_fixed`. Rows carry + `window: {kind, label}` so a cumulative row cannot be misread as periodic, and there is + NO `total` column — sliding windows overlap, so a sum of columns would double-count. + + The client sends the pids its view currently matches (the filter IS the scope); the server + intersects them with the session's book, so the request can only ever NARROW. Values are + POOLED aggregates computed by the semantic layer's own expression (the additivity law: an + average is computed at pool grain, never averaged over per-customer answers). A bucket + with no rows is 0 for a sum/count and null for anything else; a bucket that has not + STARTED yet is null for every kind — an unstarted period's YTD is unanswered, not 0. + """ + from core import measure_resolve + from harness import windows as _wn + from routes_customers import _pool_stamp, _team_agent, allowed_pids + + # Wave 16 C-TOPIC: the channel is CUSTOMER-GRAIN (measure_resolve's own grain), so the + # product surface is refused in words rather than answered with an id-space accident — + # product pids are CRC32 hashes and would mostly fall outside the customer book anyway, + # but "mostly" is not a wall. The client hides the mode for this topic; this is the + # server's half of the same refusal. + _sc = _scope_or_400(scope) + if _sc == "product" or _sc.startswith("ut_"): + raise err(400, "bad_timeseries", + "this surface has no time-series channel — measures are customer-grain") + + body = body or {} + bucket = body.get("bucket") + if bucket not in _TS_BUCKETS: + raise err(400, "bad_timeseries", "bucket must be one of week, month, quarter, year") + raw_fields = body.get("fields") + if not isinstance(raw_fields, list) or not raw_fields: + raise err(400, "bad_timeseries", "fields must be a non-empty list of field keys") + if len(raw_fields) > _TS_MAX_FIELDS: + raise err(400, "bad_timeseries", f"at most {_TS_MAX_FIELDS} fields per request") + raw_pids = body.get("pids") + if not isinstance(raw_pids, list) or not raw_pids: + raise err(400, "bad_timeseries", "pids must be a non-empty list") + if len(raw_pids) > _TS_MAX_PIDS: + raise err(400, "bad_timeseries", f"at most {_TS_MAX_PIDS} pids per request") + try: + wanted = {int(p) for p in raw_pids} + except (TypeError, ValueError): + raise err(400, "bad_timeseries", "pids must be integers") + pool = wanted & {int(p) for p in allowed_pids(session)} + if not pool: + # 403, not an empty 200: the caller asked about customers outside their book, and an + # all-zero series over nobody would read as "no activity", not "not yours". + raise err(403, "out_of_scope", "none of those customers are in your book") + + span = body.get("span") + if not isinstance(span, dict): + raise err(400, "bad_timeseries", "span must be {'lastN': n} or {'from': .., 'to': ..}") + today = time.strftime("%Y-%m-%d") + t = dt.date.fromisoformat(today) + n = span.get("lastN") + starts = [] + if n is not None: + if not isinstance(n, int) or isinstance(n, bool) or not (1 <= n <= _TS_MAX_LAST_N): + raise err(400, "bad_timeseries", f"lastN must be 1..{_TS_MAX_LAST_N}") + cur = _ts_start_of(bucket, t) + starts = [cur] + for _ in range(n - 1): + cur = _ts_prev(bucket, cur) + starts.append(cur) + starts.reverse() + else: + try: + d_from = dt.date.fromisoformat(str(span.get("from"))) + d_to = dt.date.fromisoformat(str(span.get("to"))) + except (TypeError, ValueError): + raise err(400, "bad_timeseries", "span.from/to must be ISO dates (YYYY-MM-DD)") + if d_from > d_to: + d_from, d_to = d_to, d_from + cur = _ts_start_of(bucket, d_from) + while cur <= d_to: + starts.append(cur) + if len(starts) > _TS_MAX_BUCKETS: + raise err(400, "bad_timeseries", + f"that span is more than {_TS_MAX_BUCKETS} {bucket} buckets - " + f"narrow it") + cur = _ts_next(bucket, cur) + if not starts: + raise err(400, "bad_timeseries", "the span holds no buckets") + ends = [_ts_next(bucket, s) - dt.timedelta(days=1) for s in starts] + + rt = session.runtime + if not rt.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + import modules.customer_data as cl_mod + # Read-only route: it must not consume a one-shot label-correction ack the browser has + # not seen yet (the same rule event validation follows). + ws = cl_mod.table_workspace(session.uname, consume_corrections=False) + fdefs = ws.get("fields") or {} + keys, dropped, seen = [], [], set() + for k in raw_fields: + k = str(k or "") + if not k or k in seen: + continue + seen.add(k) + fd = fdefs.get(k) + if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): + # Named, never silent: v1 eligibility is measure-backed fields only (C-TS). + dropped.append({"field": k, "reason": "not_a_measure_field"}) + continue + keys.append(k) + # ⚡ WAVE 17 R9 (amendment 2026-08-03, GRID's cross-fence ask) — A SHEET WITH NO MEASURE + # FIELDS IS NO LONGER A 400. It answers with the BUCKET GRID and no rows. + # + # Why this is the honest direction and not a loosening: the bucket starts and their labels + # are SERVER math (`_ts_start_of` / `_ts_next` / `_ts_label`), and R9 has the client + # synthesizing snapshot rows for preset columns that have no window to slide. Those rows + # must be painted under the SAME headings as everything else, so the client needs the + # columns even when the server has no series to put in them. Refusing the whole request + # meant the panel could offer nothing at all on this tenant — the shipped contract has zero + # measure-backed presets (`aios_grid.py`: "this branch is currently MEMBERLESS"), which is + # what item 5 is actually about. + # + # Nothing is invented by this: `rows` is empty and `meta.dropped` NAMES every field and why. + # A 400 is still returned for a request that is malformed (bad bucket, bad span, no fields + # at all) — this only stops treating "I asked about columns you cannot serve" as an error. + mfields, slid_keys = [], [] + for k in keys: + m = dict(fdefs[k]["measure"]) + if (m.get("window") or {}).get("kind") == "custom": + # C-TSWIN: a fixed date range cannot slide across buckets — named, never silent, + # the same posture as not_a_measure_field. + dropped.append({"field": k, "reason": "window_fixed"}) + continue + mfields.append({"key": k, "measure": m}) + slid_keys.append(k) + # Same amendment as above: every metric being fixed-range is a sheet with no SERIES, not a + # broken request. The columns still stand, and `window_fixed` still names each refusal. + if len(starts) * len(mfields) > _TS_MAX_CELLS: + raise err(400, "bad_timeseries", + "that ask is too wide - narrow the span or pick fewer metrics") + + team_id, agent = _team_agent(session) + stamp = _pool_stamp(rt, team_id, agent) + problems = [] + bucket_bounds = [(s.isoformat(), e.isoformat()) for s, e in zip(starts, ends)] + # No slidable metrics = nothing to resolve. Skipping the call rather than asking the + # resolver about an empty list keeps the memo free of a meaningless key. + answers = measure_resolve.series_values( + mfields, bucket, bucket_bounds, + team_id, frozenset(pool), today, stamp, rt.series_memo, + on_error=lambda tag, e: problems.append(str(e)[:200])) if mfields else {} + + columns = [] + for s, e in zip(starts, ends): + col = {"key": s.isoformat(), "label": _ts_label(bucket, s), + "from": s.isoformat(), "to": e.isoformat()} + if e > t: + col["partial"] = True + columns.append(col) + rows = [] + for k in slid_keys: + ans = answers.get(k) + if ans is None: + dropped.append({"field": k, "reason": "unresolvable"}) + continue + vals_by = ans.get("values") or {} + agg_kind = str(ans.get("agg") or "sum") + zero_fill = agg_kind in ("sum", "count") + vals = [] + for s in starts: + if s > t: + # C-TSWIN: an unstarted period is unanswered for EVERY agg kind — a future + # month's YTD zero-filled to 0 would read as "the year reset". + vals.append(None) + else: + vals.append(vals_by.get(s.isoformat(), 0 if zero_fill else None)) + wspec = (fdefs[k].get("measure") or {}).get("window") + wnorm = _wn.normalize(wspec) or {} + rows.append({"field": k, "label": str(fdefs[k].get("label") or k)[:120], + "agg": agg_kind, "values": vals, + "window": {"kind": str(wnorm.get("kind") or ""), + "label": _wn.label(wspec)}}) + meta = {"pool": len(pool), "today": today, "bucket": bucket} + if dropped: + meta["dropped"] = dropped + if problems: + meta["problems"] = problems[:5] + return {"columns": columns, "rows": rows, "meta": meta} + + +#: C-CAL caps (wave 17, owner item 9 / ruling R4). A month of days, a handful of metrics. +#: Every one of these is a 400 WITH ITS REASON, never a silent trim: a calendar that quietly +#: answered 20 of the 31 days asked about would paint eleven blank cells that look like days +#: with no activity. +_CAL_MAX_GROUPS = 31 +_CAL_MAX_FIELDS = 6 +_CAL_MAX_CELLS = 186 + + +@router.post("/grid/calendar_metrics") +def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "customer", + session: Session = Depends(module_gate(MODULE))): + """Per-DAY measure values with each metric's own window slid to that day (C-CAL / R4). + + ⛔ WHY THIS EXISTS AT ALL. The calendar's summary cells used to aggregate the row VALUES the + grid already held — which for a measure column means "every member's YTD **as of today**", + summed and printed under a date in March. The number was arithmetically fine and semantically + a lie: it answered a question about today while sitting in a cell labelled with another day. + R4: a metric in a day cell is computed AS OF THAT DAY. + + The shape differs from the time-series channel in exactly one way, and it is the reason this + is a separate route rather than a parameter: **every group carries its OWN pid set**. A + calendar day holds the records the date field placed there, so day-to-day the subject + changes. One `allowed_pids` for the whole request — the TS channel's shape — would compute + each day over everybody, which is a different question again. + + Static (non-measure) fields are NOT served here. They have no window to slide, so the + client's own per-day aggregation over row values stays correct for them; sending them would + invite a second implementation of arithmetic that already works. + """ + from core import measure_resolve + from harness import windows as _wn + from routes_customers import _pool_stamp, _team_agent, allowed_pids + + # The same refusal the TS channel makes, for the same reason: measures are customer-grain + # and product pids are CRC32 hashes of SKU codes. "Mostly outside the book" is not a wall. + _sc_cal = _scope_or_400(scope) + if _sc_cal == "product" or _sc_cal.startswith("ut_"): + raise err(400, "bad_calendar_metrics", + "the product surface has no measure channel yet — measures are customer-grain") + + body = body or {} + raw_groups = body.get("groups") + if not isinstance(raw_groups, list) or not raw_groups: + raise err(400, "bad_calendar_metrics", + "groups must be a non-empty list of {key: 'YYYY-MM-DD', pids: [...]}") + if len(raw_groups) > _CAL_MAX_GROUPS: + raise err(400, "bad_calendar_metrics", + f"at most {_CAL_MAX_GROUPS} days per request (one month)") + raw_fields = body.get("fields") + if not isinstance(raw_fields, list) or not raw_fields: + raise err(400, "bad_calendar_metrics", "fields must be a non-empty list of field keys") + if len(raw_fields) > _CAL_MAX_FIELDS: + raise err(400, "bad_calendar_metrics", f"at most {_CAL_MAX_FIELDS} metrics per request") + if len(raw_groups) * len(raw_fields) > _CAL_MAX_CELLS: + raise err(400, "bad_calendar_metrics", + "that ask is too wide - fewer days or fewer metrics") + + book = {int(p) for p in allowed_pids(session)} + groups, seen_days = [], set() + for g in raw_groups: + if not isinstance(g, dict): + raise err(400, "bad_calendar_metrics", "every group must be an object") + day = str(g.get("key") or "") + try: + d = dt.date.fromisoformat(day) + except (TypeError, ValueError): + raise err(400, "bad_calendar_metrics", + "every group key must be an ISO date (YYYY-MM-DD)") + if day in seen_days: + raise err(400, "bad_calendar_metrics", f"day {day} appears twice") + seen_days.add(day) + raw_pids = g.get("pids") + if not isinstance(raw_pids, list): + raise err(400, "bad_calendar_metrics", "every group needs a pids list") + try: + wanted = {int(p) for p in raw_pids} + except (TypeError, ValueError): + raise err(400, "bad_calendar_metrics", "pids must be integers") + # NARROW-ONLY, per group. The client sends what its calendar placed; the server can + # only ever remove from that, never add. + groups.append((day, d, frozenset(wanted & book))) + if sum(len(p) for _, _, p in groups) > _TS_MAX_PIDS: + raise err(400, "bad_calendar_metrics", + f"at most {_TS_MAX_PIDS} customer references per request") + + rt = session.runtime + if not rt.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + import modules.customer_data as cl_mod + ws = cl_mod.table_workspace(session.uname, consume_corrections=False) + fdefs = ws.get("fields") or {} + keys, dropped, seen = [], [], set() + for k in raw_fields: + k = str(k or "") + if not k or k in seen: + continue + seen.add(k) + fd = fdefs.get(k) + if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): + # The TS channel's vocabulary, deliberately reused rather than re-coined: the client + # already knows how to say these three words to a reader. + dropped.append({"field": k, "reason": "not_a_measure_field"}) + continue + if ((fd["measure"].get("window") or {}).get("kind") == "custom"): + dropped.append({"field": k, "reason": "window_fixed"}) + continue + keys.append(k) + if not keys: + raise err(400, "bad_calendar_metrics", + "none of the requested fields are measure fields with a window that can " + "slide to a day") + + today = time.strftime("%Y-%m-%d") + t = dt.date.fromisoformat(today) + team_id, agent = _team_agent(session) + stamp = _pool_stamp(rt, team_id, agent) + problems = [] + values = {k: {} for k in keys} + for day, d, pids in groups: + # A day that has not happened is unanswered for EVERY aggregate kind — the unstarted + # bucket law at day grain. Answering 0 would say "we sold nothing", which is a claim + # about a day nobody has lived through yet. + if d > t or not pids: + for k in keys: + values[k][day] = None + continue + answers = measure_resolve.series_values( + [{"key": k, "measure": dict(fdefs[k]["measure"])} for k in keys], + "day", [(day, day)], team_id, pids, today, stamp, rt.series_memo, + on_error=lambda tag, e: problems.append(str(e)[:200])) + for k in keys: + ans = answers.get(k) + if ans is None: + values[k][day] = None + continue + vals_by = ans.get("values") or {} + zero_fill = str(ans.get("agg") or "sum") in ("sum", "count") + values[k][day] = vals_by.get(day, 0 if zero_fill else None) + + for k in keys: + if all(v is None for v in values[k].values()): + # Every day unanswerable is a FIELD-level failure, and saying so is the difference + # between "no activity that month" and "this metric could not be computed". + dropped.append({"field": k, "reason": "unresolvable"}) + out = {"values": values, "today": today, + "windows": {k: {"kind": str((_wn.normalize( + (fdefs[k].get("measure") or {}).get("window")) or {}).get("kind") or ""), + "label": _wn.label((fdefs[k].get("measure") or {}).get("window"))} + for k in keys}} + if dropped: + out["dropped"] = dropped + if problems: + out["problems"] = problems[:5] + return out + + +#: ⭐⭐ THE BULK CELL DOOR'S OWN CEILING, and it is REPORTED rather than silently enforced. The +#: standing no-cap rule governs data read from a connected source; this is a WRITE of team-typed +#: values, so a bound is right — but R6's second sentence still binds, which is why exceeding it +#: is a 400 naming the number and the count, never a truncation. +#: Sized against the real job with headroom: the 2027 catalog is 1,397 rows. +MAX_BULK_ROWS = 10_000 + + +@router.post("/grid/bulk-cells") +def bulk_cells(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """`{scopeKey, rows: {"": {key: value}}}` → shared cells on many rows, ONE transaction. + + ⭐⭐ WHY THIS EXISTS AT ALL, because "we already have `/grid/events`" is the obvious objection. + That door is capped at `_MAX_EVENTS = 24` (it is sized for the client's resend window), so a + 1,397-row import is ~59 sequential POSTs against ONE JSON document — and this repo has a + MEASURED scar for exactly that: 18 writes against one document under the store's coalescing + single-flight landed **zero** while answering 200 eighteen times. Raising `_MAX_EVENTS` would + have widened the browser's own resend window to fix a script's problem. One door, one + transaction, is the honest shape. + + ⛔ TENANT-WIDE KEYS ONLY, AND THAT IS THE POINT RATHER THAN A LIMITATION. It writes through + `shared_overlay`, which every account in the tenant reads. A per-user bulk write would be a + contradiction: nobody imports 1,397 rows of the team's work so that one account can see it — + that is precisely the failure owner ruling R6 was made to avoid. A key that is not declared + `shared: true` in the canonical contract is REFUSED BY NAME. + + ⛔ ADMIN ONLY. This mutates data every account in the tenant reads, in bulk, in one call. + D-172 already records that the shared stratum has an asymmetric wall (anyone may create a + tenant-wide column, only the creator or an admin may delete one); a new door does not get to + inherit the loose half of an asymmetry somebody already flagged. + """ + import core.shared_overlay as shared_overlay + import modules.product_data as pd + import core.perm_scope as perm_scope + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + scope = _scope_or_400((body or {}).get("scopeKey")) + if scope != "product": + # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. + raise err(400, "scope_not_bulk_writable", + f"bulk cell writes are available on the product database; '{scope}' has no " + f"tenant-wide cell stratum") + session.require(PRODUCT_MODULE) + if not session.admin: + raise err(403, "admin_only", + "a bulk write changes values every account in this workspace reads") + + rows = (body or {}).get("rows") + if not isinstance(rows, dict): + raise err(400, "bad_rows", "expected {rows: {\"\": {field: value}}}") + if len(rows) > MAX_BULK_ROWS: + raise err(400, "too_many_rows", + f"at most {MAX_BULK_ROWS} rows per request; this one carried {len(rows)}") + + pids, _team, _src, fields_base = scoped_pool(session) + allowed = {int(p) for p in pids} + shared_keys = set(pd.SHARED_KEYS()) + hidden = perm_scope.hidden_keys(session.user, PRODUCT_MODULE, fields_base) + + clean, unknown_pid, refused_keys = {}, [], {} + for raw_pid, values in rows.items(): + try: + pid = int(raw_pid) + except (TypeError, ValueError): + unknown_pid.append(str(raw_pid)[:40]) + continue + # ⚠ THE POOL IS THE WALL. `scoped_pool` is the same predicate the read door uses, so a + # caller cannot write a row they could not see — including a row in another BU. + if pid not in allowed: + unknown_pid.append(str(raw_pid)[:40]) + continue + keep = {} + for key, value in dict(values or {}).items(): + key = str(key) + if key not in shared_keys: + refused_keys.setdefault("not_shared", set()).add(key) + continue + if key in hidden: + refused_keys.setdefault("hidden_by_permissions", set()).add(key) + continue + if not isinstance(value, (str, int, float)) or isinstance(value, bool): + refused_keys.setdefault("unsupported_value", set()).add(key) + continue + keep[key] = value + if keep: + clean[pid] = keep + + written = shared_overlay.put_rows(pd.TABLE_KEY, clean, st=pd.TABLE_OPS.st) if clean else {} + out = { + "rows_written": len(written), + "cells_written": sum(len(v) for v in written.values()), + "rows_requested": len(rows), + } + # ⭐ R6's second sentence: what did NOT land, and why. A bulk door that reports only its + # successes is how 701 unmatched SKUs disappear quietly. + if unknown_pid: + out["rows_not_in_your_pool"] = {"count": len(unknown_pid), + "sample": sorted(unknown_pid)[:10]} + if refused_keys: + out["refused_fields"] = {reason: sorted(keys) + for reason, keys in refused_keys.items()} + return out + + +@router.post("/grid/events") +def grid_events_route(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """`{events: []}` → `{results, doc?, toast?}`. + + Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the + same sessions the read door (`/workspace`) admits, or a tenant without the customer module + can SEE its own user tables and not write to them.""" + from core import grid_events + from routes_customers import grid_assembly + + events = (body or {}).get("events") + if events is None and isinstance(body, dict) and body.get("type"): + events = [body] # a single event object, the legacy shape + if not isinstance(events, list): + raise err(400, "bad_events", "expected {events: [...]}") + if len(events) > _MAX_EVENTS: + raise err(400, "too_many_events", + f"at most {_MAX_EVENTS} events per request (the client's resend window)") + + # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be + # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a + # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on + # what a scope is, or the surface you read is not the surface you wrote. + scope = _scope_or_400((body or {}).get("scopeKey")) + # Wave 21 C4 — same per-scope gate as /workspace (read and write doors must agree). + if not (scope == "product" or scope.startswith("ut_")): + session.require(MODULE) + # This assembly validates the write; it is not a payload the browser will render. Leave + # one-shot field-name correction acks queued for the subsequent /workspace refresh. + # Wave 16 C-TOPIC: the PRODUCT topic gets the product assembly — product field contract, + # product pids, and (below) the product TABLE OPS, so a product event is validated against + # and lands in the product bucket. The measure/cohort context is honestly EMPTY there: + # `clean_measure_field` refuses measure creates on this surface by construction (the + # customer-grain descope), which is the fail-closed shape, not an accident. + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, product_assembly + + session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4) + g = product_assembly(session, consume_corrections=False) + elif scope.startswith("ut_"): + # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the + # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here). + # ⭐⭐ WAVE 30 / W30-T30 (owner item 2: *"when I click hide fields it crash… no matter the + # size"*). This used to be `ut_assembly(...)`, which builds the whole table to validate a + # write it then throws away: `scoped_pool` allocates a dict per row and sorts them, so ONE + # hide-fields checkbox rebuilt ~33k order rows before the event was even dispatched. + # ⛔ NOTHING IS VALIDATED LESS. The comment above still holds — this assembly is the + # permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys + # this route reads, with a pid set derived from exactly the row ids `scoped_pool` would + # have kept. What it does not do is materialise the rows nobody here looks at. + from routes_tables import ut_write_ctx + + g = ut_write_ctx(session, scope) + # ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the + # half that is easy to skip because the code already "fails closed" without it. When a + # read-through grid's population exceeds one window the pid set is EMPTY, and every + # pid-bearing handler in `grid_events` then answers `False` — `overlay_patch` and + # `add_to_list` both `return False` for a pid not in `allowed_pids`. On the wire that is + # `rerender: false` and HTTP 200: a write the user watched succeed, that did nothing + # ([[lost-write-looks-like-failed-read]]). Schema-only events — hide a field, save a view, + # rename a column — name no pid and are untouched, which is D-170. + # ⚠ THE TEST IS THE EVENT'S OWN KEYS, not a kind list: a new pid-bearing event type would + # otherwise inherit the silent no-op the day it is added. + if g.get("limits"): + named = [e for e in events + if isinstance(e, dict) and (e.get("pid") is not None or e.get("pids"))] + if named: + lim = g["limits"][0] + raise err(409, "pid_scope_unresolved", + f"this database is served through the connector mirror and its rows " + f"cannot be listed in one window, so a change addressed to particular " + f"records ({len(named)} of {len(events)} here) cannot be admitted — " + f"{lim.get('cause') or 'the row set is unresolved'}. " + f"{lim.get('recommendation') or ''}".strip()) + else: + g = grid_assembly(session, scope=scope, consume_corrections=False) + # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`, + # `clean_measure_field` had an empty admission list and every measure-column create over + # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every + # measure CONDITION out of a saved view. The embed always passed these; the API adapter + # simply had not been given them — the standalone shell could read measures it could + # never write. + ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, + measure_keys=frozenset(m["key"] for m in g["measures"]), + resolved_ids=frozenset(g["measure_sets"]), + cohort_ids=frozenset(c["id"] for c in g["lists"]), + measure_offer=tuple(g["measures"]), + visible_views=tuple(g["views"])) + + # Per-event results so the client can tell which of a batch landed — the component's own + # bridge has no response channel at all, so this is strictly more than the embed gets. + results = [] + try: + for one in events: + eid = str(one.get("id") or "") if isinstance(one, dict) else "" + # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler + # appends to a shared list, so the refusals belonging to THIS event are exactly the + # ones that appeared across THIS call. Reading the list once after the loop would + # answer "the batch was refused" and leave the caller to guess which member, which is + # the same class of unfalsifiable answer the channel exists to end. + _before = len(ctx.out.refusals) + rerender = grid_events.handle_one(one, ctx) + row = {"id": eid, "rerender": bool(rerender)} + mine = ctx.out.refusals[_before:] + if mine: + # `refused` is the SHAPE a caller branches on; the first reason is the one that + # stopped this write (a handler returns at its first refusal). + row["refused"] = mine[0] + results.append(row) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — none of your changes were saved") + + out = {"results": results, "rerender": any(r["rerender"] for r in results)} + if ctx.out.refusals: + # ⚠ ALSO AT THE TOP LEVEL, because a batch that was wholly refused must not read as a + # batch that wholly landed. A caller that only checks the envelope still learns something + # is wrong, and a caller that walks `results` learns exactly which member. + out["refusals"] = list(ctx.out.refusals) + if ctx.out.doc is not None: + # ⭐ C4 / W30-T27 — `docPayload` IS THE NAME THE CLIENT ALREADY DECLARES. `types.ts` has + # carried `docPayload?: {pid, docId, name, mime, data_b64}` since C5, and `Documents.tsx` + # matches it against the fetch it is waiting on — while this route has been answering + # `doc`, which `apiBridge.ts` deliberately drops. One object, emitted under the name the + # consumer looks for, so F's wiring needs no translation step to get wrong. + # ⚠ `doc` stays for one wave: nothing in the client reads it, but a gate might, and + # removing a key to save six bytes is not worth a red nobody predicted. + out["doc"] = out["docPayload"] = ctx.out.doc + if ctx.out.toast is not None: + out["toast"] = ctx.out.toast + + # ── ⭐ owner item 2 (2026-08-03): THE NEW MEASURE COLUMN'S VALUES, ONE ROUND TRIP SOONER ── + # + # Creating a measure column cost the browser TWO sequential trips before a single number + # appeared: this one to persist the field, then a whole `/workspace` to compute it. The + # second cannot start until the first lands (the resolver reads the PERSISTED field), so the + # wait was structural, not slow code — the owner's "it takes some time for the data to + # populate". The values are computed here instead, immediately after the write, and ride + # this response. + # + # ⚠ IT COSTS NOTHING EXTRA TO COMPUTE. The expensive part is one DuckDB aggregate over the + # book, and `rt.measure_memo` is keyed on (pool stamp, scope, pool, measure, window) — so + # the `/workspace` re-read that still follows HITS the memo instead of doing this work. The + # query happens once either way; only its position moved. + # + # ⚠ NARROW ON PURPOSE. Gated to an actual measure-column write, so an overlay edit or a + # cohort add — the overwhelming majority of events — never pays for a second assembly. + # + # ⚠ AND IT IS A SHORTCUT, NOT A PATH. Any failure is swallowed: `WORKSPACE_STALE` still + # fires from `rerender`, and the re-read still delivers these values exactly as it does + # today. Nothing depends on this having worked. + if out["rerender"] and scope in ("customer", "cohort") and any( + isinstance(e, dict) and e.get("type") == "field_upsert" + and str(((e.get("field") or {}) if isinstance(e.get("field"), dict) else {}) + .get("key") or "").startswith("measure_") + for e in events): + try: + fresh = grid_assembly(session, scope=scope, consume_corrections=False) + out["derived"] = {str(pid): cells for pid, cells in fresh["derived"].items()} + except Exception: + pass + # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped + # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an + # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views, + # folders, cohorts) is re-read from the store on the next request. An earlier version cleared + # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh + # data that was never in it. + return out + + +# ── CONTRACT C10 (W41-T20): THE PIVOT ───────────────────────────────────────────────────────── +# ⭐⭐ R10 — *"the products ORDERED BY the filtered customers"*, over a time window. One door, and +# it answers with the TARGET database's OWN rows and OWN field contract, so the client paints a +# second grid rather than learning a bespoke payload shape. +# +# ⛔⛔ THERE IS NO LINK FIELD FOR THIS PAIR, AND W41-T19 REPORTED THAT RATHER THAN FAKING ONE. +# Odoo declares no `res.partner` relation on `product.product` (read from `ir.model.fields`); the +# only path is `sale.order.line`, which carries BOTH endpoints (`order_partner_id`, `product_id`) +# and is therefore TWO HOPS. The link-bag vocabulary has no `through` leg, and `ut_odoo_order_lines` +# holds zero rows because it is read-through by design. The blocker is recorded at +# `odoo_relational.py::_PRODUCT_LINK_BLOCKER`. +# +# ⭐ SO THIS ROUTE TAKES THE SEMANTIC PATH, and that is the choice, stated rather than discovered: +# `model/topics/sales_lines.yml` already carries `order_partner` AND `product_code` as dims of ONE +# fact table, which IS the reachability edge, and `semantic.store_query(filters={dim: ids})` emits +# an UNCAPPED `IN (?,…)` over them. Confirmed orders only, the wholesale teams and the GIFTWARE +# DEALS exclusion all come from that topic's `scope_sql`, so the ticket's "confirmed orders only" +# is enforced by the ONE definition of it rather than by a second copy written here. +# ⚠ `viaField` therefore RIDES THE WIRE AND IS ADVISORY for this pair (C10, and lane D types +# against it). A door that refused an unresolvable `viaField` would refuse R10's headline case. +# +# ⛔ AND IT IS STILL NOT CHAINING (R12). `path` is TWO databases. `sales_lines` is the EDGE, not a +# third stop: none of it reaches the client, no filter over it is accepted, and no caller can name +# it. Putting it in `path` would read as exactly the third database R12 forbids. + +#: R10's window control, as the WIRE names lane D emits, mapped onto `harness.windows`' CLOSED +#: vocabulary. ⛔ THE ARITHMETIC IS NOT RESTATED HERE, and that is the whole point of the dict: +#: "the last 12 months" already has exactly one definition in this codebase (`windows.resolve`'s +#: `ltm` = 365 days back, both ends inclusive, which `core.periods.ltm` agrees with to the day), +#: and a second one written beside a pivot is the two-definitions-of-one-fact defect +#: [[date-window-vocabulary]]. This is a RENAME and nothing else. `all` resolves to `(None, None)`. +PIVOT_WINDOWS = {"all": "all_time", "last12m": "ltm", "ytd": "ytd"} + +#: The DECLARED pivot routes: `(source, target) -> the edge that answers it`. A pair with no entry +#: is a 400, never an empty grid — "there is no way to get there" and "nothing is related" are two +#: different answers and only one of them is true. +#: +#: ⚠ `source_key` IS ASSERTED, NOT DECORATION. `filter_eval.visible_pids` reads `row['pid']` and +#: nothing else, so a future route whose source identity is NOT `pid` would silently pivot on the +#: wrong column. The route refuses instead. ⭐ For `customer_data` this is exactly right and W41-T19 +#: measured it: `partner_id` is declared `derived` and is on 0 of 3,643 rows, while `pid` IS the +#: `res.partner` id. `from: "partner_id"` resolves 0 rows with nothing red. +#: +#: ⚠ `target_dim` IS THE SKU CODE, NOT `product_id`, AND THE OBVIOUS CHOICE IS THE WRONG ONE. +#: `product_code` is written to reproduce the product grid's identity exactly, `pid:` +#: fallback included; `product` (`l.product_id`) splits a re-SKUed pair whose archived half has no +#: grid row at all. MEASURED on this store: over the wholesale scope, grouping by `product_id` +#: gives 3,360 buckets and by `product_code` gives 3,352 — 8 codes carry two ids each, and it is +#: the archived id of each pair that no `product_data` row claims (that module keeps archived +#: products out, by R12). Joining on the id therefore loses 8 real products while looking fine. +_PIVOT_ROUTES = { + ("customer_data", "product_data"): { + "topic": "sales_lines", + "measure": "revenue", + "scope_key": "customer", # the cohort bucket the SOURCE tree's cohort leaves name + "source_dim": "order_partner", + "source_key": "pid", + "target_dim": "product_code", + "target_key": "code", + }, +} + + +def _pivot_today(): + """The tenant's today, ISO. One clock, so the window and any date leaf agree.""" + import core.periods as periods + + return periods.today().isoformat() + + +def _pivot_reply(source, target, rows, fields, source_count, target_count, + bridge_count=0, refusal=None, limits=()): + """THE C10 ENVELOPE, in one place so every exit from the route emits the same keys. + + ⛔ `refusal` IS NOT AN ERROR AND IT IS NOT AN EMPTY RESULT. A reader walled out of either side + of the join gets 200 with `rows: []` AND a reason, because a bare empty grid reads as "no + related records", which is a claim about the data rather than about the reader. A genuinely + empty related set carries neither a refusal nor a limit: that is an honest zero. + + ⚠ `bridgeCount` IS THE HONESTY CHANNEL AND IT IS ADDITIVE. It is how many distinct related + identities the edge found; `targetCount` is how many of those resolved to a row this reader may + see. They differ when a related product has no ACTIVE catalogue record (archived, kept out of + the grid by R12), and without the pair that difference would be invisible. + """ + out = { + "rows": list(rows or ()), + "fields": list(fields or ()), + "sourceCount": int(source_count), + "targetCount": int(target_count), + "bridgeCount": int(bridge_count), + "path": [source, target], + } + if refusal is not None: + out["refusal"] = refusal + if limits: + out["limits"] = list(limits) + return out + + +def _pivot_side(session, key): + """`((fields, rows), None)` for ONE side of a pivot, or `(None, {reason, subject})`. + + ⭐⭐ R12: THE PERMISSION WALL APPLIES ON BOTH SIDES OF THE JOIN, and this is the function that + makes that structural rather than remembered. Every side goes through the SAME two gates in the + same order: the tenant/account module gate (`session.require`, which is what the nav asks) and + then contract C1's one door (`perm_scope.scoped_table` / `scoped_fields`), which applies the row + wall and the field wall and hands back only what this principal may see. + + ⛔ EVERY REFUSAL IS RETURNED, NEVER RAISED. The caller turns it into a 200 with a reason on it; + see `_pivot_reply`. + """ + from fastapi import HTTPException + + import core.perm_scope as perm_scope + + try: + session.require(key) + except HTTPException as exc: + detail = exc.detail if isinstance(exc.detail, dict) else {} + message = ((detail.get("error") or {}).get("message") + or f"your account may not open {key}") + return None, {"reason": message, "subject": key} + try: + fields = perm_scope.scoped_fields(session.user, key, st=session.runtime) + rows = perm_scope.scoped_table(session.user, key, st=session.runtime) + except perm_scope.Denied as exc: + return None, {"reason": str(exc), "subject": key} + except perm_scope.Unresolvable as exc: + # Standing rule 1's second sentence, carried through unchanged: the cause AND the + # recommendation, never a short answer with nothing red beside it. + return None, {"reason": f"{exc.cause}. {exc.recommendation}", "subject": key} + except perm_scope.UnknownTable as exc: + return None, {"reason": str(exc), "subject": key} + return (fields, rows), None + + +def _pivot_unanswerable(nodes, columns): + """The ACTIVE source leaves this door cannot resolve, by kind: `measure`, `rank`, or neither. + + ⛔ WHY THIS EXISTS RATHER THAN LETTING THE ENGINE FAIL CLOSED. `filter_eval` answers an + unresolved measure or rank leaf with FALSE for every row, deliberately (the widening sin is the + worse one). On this route that lands as a pivot of zero customers and therefore zero products, + which paints as "this selection has ordered nothing" — a claim about the business made out of a + missing input. So the leaf is named instead. An INACTIVE leaf asks nothing and is ignored here + exactly as the engine ignores it; refusing on one would break a half-built condition. + """ + from harness import filter_eval + + found, stack = set(), list(nodes or ()) + while stack: + node = stack.pop() + if not isinstance(node, dict): + continue + if isinstance(node.get("children"), list): + stack.extend(node["children"]) + continue + if not filter_eval.is_rule_active(node, columns): + continue + if node.get("window") is not None: + found.add("measure") + elif node.get("op") in filter_eval.RANK_OPS: + found.add("rank") + return sorted(found) + + +@router.post("/grid/pivot") +def grid_pivot(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """C10 — `{source, filters, filterConj, target, viaField, window}` -> the TARGET's rows. + + Answers `{rows, fields, sourceCount, targetCount, bridgeCount, path, refusal?, limits?}`. + + ⭐ BOTH COUNTS, ALWAYS. `sourceCount` is how many source records the filter selected; + `targetCount` is how many related target rows came back. Neither is derived from the other and + neither is a window count: nothing here is windowed, because standing rule 1 forbids a cap on + connected-source data and the whole related set of a connector-backed grid is small enough to + serve (5,873 products, 3,643 customers on tenant #0). A limit that ever DID apply would arrive + as a reported `limits` entry carrying its cause and a recommendation, never as a short list. + """ + import core.perm_scope as perm_scope + import harness.semantic as semantic + from harness import filter_eval, windows + + body = body if isinstance(body, dict) else {} + source = str(body.get("source") or "").strip() + target = str(body.get("target") or "").strip() + window = str(body.get("window") or "all").strip() or "all" + filters = body.get("filters") if isinstance(body.get("filters"), list) else [] + filter_conj = "or" if body.get("filterConj") == "or" else "and" + + if window not in PIVOT_WINDOWS: + raise err(400, "bad_window", + f"window must be one of {', '.join(sorted(PIVOT_WINDOWS))}. Refusing to guess " + f"which period was meant, because guessing 'all time' would widen the answer " + f"under a count nobody would doubt") + route = _PIVOT_ROUTES.get((source, target)) + if route is None: + raise err(400, "no_pivot_route", + f"there is no declared way to get from '{source}' to '{target}'. " + f"Declared: " + + "; ".join(f"{a} to {b}" for a, b in sorted(_PIVOT_ROUTES))) + + # ── THE WALL, ON BOTH SIDES OF THE JOIN (R12) ──────────────────────────────────────────── + src, refusal = _pivot_side(session, source) + if refusal is not None: + return _pivot_reply(source, target, [], [], 0, 0, refusal=refusal) + tgt, refusal = _pivot_side(session, target) + if refusal is not None: + # ⛔ NO FIELDS EITHER. A reader who may not open the target may not learn its columns. + return _pivot_reply(source, target, [], [], 0, 0, refusal=refusal) + src_fields, src_rows = src + tgt_fields, tgt_rows = tgt + + today = _pivot_today() + columns = filter_eval._columns_map(src_fields) + unanswerable = _pivot_unanswerable(filters, columns) + if unanswerable: + return _pivot_reply( + source, target, [], tgt_fields, 0, 0, + refusal={"subject": source, + "reason": f"this filter carries a {' and a '.join(unanswerable)} condition, " + f"which is resolved by the grid that owns it and cannot be " + f"answered here. Pivot from a view filtered on columns and " + f"cohorts, or narrow the source grid first and pivot from that"}) + + # ⭐ COHORT LEAVES ARE RESOLVED, NOT LEFT TO FAIL CLOSED. `filter_eval` answers a cohort it was + # given no membership for with False for every row, which on this route would read as "these + # customers ordered nothing". Same projection `aios_grid.workspace_wire` builds, so a cohort + # means here what it means on the grid the filter was written on. + import modules.cohort as cohort_mod + + pool_pids = frozenset(r.get("pid") for r in src_rows) + cohort_sets = { + cid: {p for p in (c.get("members") or ()) if p in pool_pids} + for cid, c in cohort_mod.scoped(route["scope_key"]).visible( + session.uname, pool_pids).items()} + + if route["source_key"] != "pid": + raise err(500, "pivot_source_key", + f"this pivot names '{route['source_key']}' as the source identity and the row " + f"engine reads 'pid'. Refusing rather than pivoting on the wrong column") + member = sorted({p for p in filter_eval.visible_pids( + {"conj": filter_conj, "nodes": filters}, src_rows, src_fields, + ctx=filter_eval.EvalCtx(cohort_sets=cohort_sets, today=today)) + if isinstance(p, int)}) + source_count = len(member) + if not member: + # An honest zero: the filter selected no source records, so nothing is related to them. + return _pivot_reply(source, target, [], tgt_fields, 0, 0) + + date_from, date_to = windows.resolve({"kind": PIVOT_WINDOWS[window]}, today) + # ⛔ THE READER'S BU RIDES INTO THE EDGE. `sales_lines.scope_sql` bakes in the wholesale teams + # and the Amazon exclusion; it does not know who is asking. Without this, a Fisch-scoped reader + # whose customers also order on Royal would be shown products related through ROYAL lines: both + # endpoints permitted, the EDGE between them not. Strict BU isolation is about the edge too. + team_id, _agent = perm_scope.derive_pool_scope(session.user, source) + try: + found = semantic.store_query( + route["topic"], [route["measure"]], group_by=[route["target_dim"]], + filters={route["source_dim"]: member}, + date_from=date_from, date_to=date_to, team_id=team_id, today=today, + limit=semantic.MAX_GROUPS) + except semantic.ModelError as exc: + raise err(503, "pivot_unavailable", str(exc)) + + # ⛔ `truncated` IS READ, NOT ASSUMED FALSE. `store_query` returns groups plus a flag, and a + # grouped query's groups ARE the answer, so a dropped one is missing data with nothing to + # notice ([[no-unverifiable-aggregates]]). Standing rule 1: report the cause and a fix. + if found.get("truncated"): + raise err(409, "pivot_truncated", + f"this pivot found more than {semantic.MAX_GROUPS:,} related records, which is " + f"more than one answer can carry. Nothing was truncated and nothing is being " + f"shown, because a short list here would understate the relationship while " + f"looking complete. Narrow the source filter or the window and ask again") + + dim, tkey = route["target_dim"], route["target_key"] + # A NULL group is a line whose product record is gone; it can match no target row, and it is + # dropped here rather than left to compare `'None'` against a real code. + keys = {str(r.get(dim)) for r in found["rows"] if r.get(dim) is not None} + rows = [r for r in tgt_rows if str(r.get(tkey)) in keys] + return _pivot_reply(source, target, rows, tgt_fields, + source_count, len(rows), bridge_count=len(keys)) + + +# ── CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS ─────────────────────────────────── +# ⭐⭐ R6 — *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table` +# is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never +# imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid` +# behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as +# `routes_odoo_tables` declares connected tables to `user_tables.register_connected`. +# +# ⛔ REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files +# are outside wave 36's lane-C fence. The readers themselves are three lines each and call the +# SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no +# second scope derivation — only a second CALLER of the one that exists. +# +# ⚠ AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` + +# `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and +# `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done +# here, because neither file is in this fence. What ships now is the arm the wave is load-bearing +# on — every `ut_*` database, plus E's sandbox — and a topic arm that is REAL rather than stubbed, +# so `scoped_table`'s topic leg is exercised by the product instead of only by a gate. +def _topic_rt(st, module): + """The tenant runtime a topic pool must be built against, or a REPORTED refusal. + + ⛔ A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the + default" without picking a tenant at random — which on this box is tenant #0's PRODUCTION + data. Standing rule 1's second sentence: say why, and say what to do instead. + """ + if st is None: + import core.perm_scope as perm_scope + raise perm_scope.Unresolvable( + subject="rows", effect="unreadable", + cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant " + f"runtime was passed", + recommendation="pass the session's runtime as `st=`. A topic pool cannot be " + "resolved without knowing which tenant is asking") + return st + + +def _customer_rows(table_key, user, st): + """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" + import aios_grid + import core.perm_scope as perm_scope + from routes_customers import _pool_for + + rt = _topic_rt(st, table_key) + team_id, agent = perm_scope.derive_pool_scope(user, table_key) + return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) + + +def _product_rows(table_key, user, st): + """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a + BU-pinned reader gets that BU's field contract rather than the consolidated one.""" + import core.perm_scope as perm_scope + from routes_products import _pool_for, pd_fields + + rt = _topic_rt(st, table_key) + team_id, _agent = perm_scope.derive_pool_scope(user, table_key) return pd_fields(consolidated=team_id is None, st=st), _pool_for(rt, team_id) - - -def _register_topic_rows(): - """Declare both topic readers to C1. Called at import; returns the registered key set. - - ⚠ THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at - module import, and `from routes_products import MODULE` here would pull a sibling router in - before its own imports have settled. Every other cross-router reference in this file is lazy - for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot - drift into naming a topic that does not exist. - """ - import core.perm_scope as perm_scope - - perm_scope.register_rows(_customer_rows, MODULE) - return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE) - - -#: ⚠ `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE` -#: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same -#: thing and be a different one — [[constant-two-features-share]] waiting to happen. -_PRODUCT_MODULE = "product_data" - -_C1_ROW_SOURCES = _register_topic_rows() + + +def _register_topic_rows(): + """Declare both topic readers to C1. Called at import; returns the registered key set. + + ⚠ THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at + module import, and `from routes_products import MODULE` here would pull a sibling router in + before its own imports have settled. Every other cross-router reference in this file is lazy + for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot + drift into naming a topic that does not exist. + """ + import core.perm_scope as perm_scope + + perm_scope.register_rows(_customer_rows, MODULE) + return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE) + + +#: ⚠ `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE` +#: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same +#: thing and be a different one — [[constant-two-features-share]] waiting to happen. +_PRODUCT_MODULE = "product_data" + +_C1_ROW_SOURCES = _register_topic_rows() diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 0d8534c02e4acf8e5318379694ff5a39602887f5..4ef61ca06030bf7abf2238ae1421cbde6686560d 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -1,1073 +1,1073 @@ -"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). - -Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER -return a decrypted field — list rows carry a masked preview, and the decrypt function is a -connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's -env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** -count (rows holding overlay data whose pids the current pool no longer serves; counted and -drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly -in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with -the keychain cutover wave R3 staged. -""" -import os - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session -# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door -# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall -# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per -# entry instead of per endpoint. `session.admin` is still what business-wide requires. - -router = APIRouter(prefix="/api/v1") - -#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by -#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another -#: freezes nothing while reporting success, which is the shape of the bug D-10 books. -from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 - ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) -#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED -#: odoo connector is paused; read by the pool path while paused; survives a Space restart. -SNAPSHOT_KEY = "connector_snapshots" - -#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. -#: -#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves -#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and -#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's -#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's -#: connection, and handing it to another tenant is the leak this method exists to prevent."* -#: -#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and -#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value -#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, -#: the gate reds instead of the product quietly disagreeing with itself. -ENV_ODOO_TENANT = "royal-imports" - - -def env_odoo_available(rt): - """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) - - ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. - `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant - guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was - `connected` and offered "Manage keys" for a credential belonging to another company. This - module asked the same question correctly two functions below, which is the whole shape of - [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in - as the second tenant. - - ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which - source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, - wrong for "should the environment row be listed at all", which is what the connectors pane - needs in order to show an inactive env source beside an active keychain one. - - Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is - how another tenant's environment got reported as this tenant's connection in the first place. - """ - try: - if not (os.environ.get("ODOO_URL") or "").strip(): - return False - return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT - except Exception: # noqa: BLE001 - return False - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. -# Every keychain entry and connector carries a scope, selectable per connection; business-wide is -# admin-only and applies to every user in the tenant."* -# -# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file -# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is -# an access rule, not a secret, so it lives in the layer that already decides who may ask. -# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves -# cannot drift into two vocabularies. -SCOPES = ("business", "personal") -#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before -#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. -#: Materialising it would be a migration nobody authorised and would rewrite the store on the next -#: read [[a-migration-that-runs-on-the-next-write]]. -DEFAULT_SCOPE = "business" -#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. -#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean -#: the same thing and there is only one way to spell the common case. -SCOPE_KEY = "keychain_scopes" - -#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. -#: -#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* -#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is -#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they -#: would silently become the credential the entire workspace's databases are built from, which is -#: a credential elevation wearing a scope picker. -#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes -#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is -#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door -#: from the only side B owns, and the resolver-side guard is booked for A. -TENANT_WIDE_TYPES = ("odoo", "meta_ads") - - -def clean_scope(raw, default=DEFAULT_SCOPE): - """A scope word from the wire, or None when the caller said something we do not speak. - - Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole - reason this returns None rather than falling back: a typo'd `"personel"` silently becoming - business-wide is exactly the failure a scope picker exists to prevent. - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default - got = str(raw).strip().lower() - return got if got in SCOPES else None - - -def _scope_rows(rt): - try: - return dict(rt.get(SCOPE_KEY) or {}) - except Exception: # noqa: BLE001 - return {} - - -def entry_scope(rt, entry_id, rows=None): - """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a - census does not re-read the store once per entry.""" - r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) - if not isinstance(r, dict): - return DEFAULT_SCOPE, "" - return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) - - -def may_see(scope, owner, uname): - """R4's visibility rule: business-wide is everyone's, personal is its owner's. - - ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the - per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token - is identity, not infrastructure"*). An admin who could read every member's personal credential - would make "personal" a label rather than a boundary. - """ - return scope != "personal" or str(owner) == str(uname) - - -def visible_entries(rt, uname, is_admin=False): - """This USER's view of the keychain: every business entry plus their own personal ones, each - row carrying its `scope` and `owner` so no caller has to ask a second time. - - ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can - hold a connection of their own; it does not hand them four characters of the workspace's Odoo - key. So a business row a member did not create arrives WITHOUT `preview` — they can see that - the connection exists and is theirs to use, which is the whole of what R4 grants. The default - is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. - """ - rows = _scope_rows(rt) - out = [] - for e in _kc().list_entries(rt): - scope, owner = entry_scope(rt, e["id"], rows) - if not owner: - owner = str(e.get("createdBy") or "") - if not may_see(scope, owner, uname): - continue - row = {**e, "scope": scope, "owner": owner} - if not (is_admin or str(owner) == str(uname)): - row["preview"] = "" - out.append(row) - return out - - -def _write_scope(rt, entry_id, scope, owner): - """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the - default, so the store holds one spelling of the common case.""" - def _up(cur): - if scope == DEFAULT_SCOPE: - cur.pop(str(entry_id), None) - else: - cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} - return cur - - rt.update(SCOPE_KEY, _up, flush="sync") - return True - - -def _may_touch(session, row): - """May this session change or delete `row`? An admin owns the business-wide ones; a member - owns their own personal ones. Anything else is not theirs to move.""" - if row.get("scope") == "personal": - return str(row.get("owner") or "") == str(session.uname) - return bool(session.admin) - - -def _kc(): - import core.keychain as keychain - return keychain - - -def _resolved_odoo_key(rt): - """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the - shapes this module needs: the display string (`env` / `keychain:`) and the key the pause - flag is stored under. - - ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, - beside the `odoo_source()` it must agree with. This function had its own copy of the same - three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and - a second copy is exactly how a pause flag comes to be written against one resolution and read - against another, freezing nothing while the UI reports success. The two SHAPES stay here - because they are this module's presentation concern; the DECISION does not. - """ - flag_key = rt.odoo_flag_key() - if not flag_key: - return None, None - return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key - - -def odoo_paused(rt): - """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry - that is not the resolved source freezes nothing — it serves nothing. - - ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the - measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question - the customer pool does. This name stays because `routes_customers`, `routes_products` and - `verify_api` all call it — moving the logic without moving the door keeps one answer and - costs no caller a change. - """ - return bool(rt.odoo_paused()) - - -def _snap_scope_key(team_id, agent): - return f"t={team_id}|a={agent}" - - -def load_pool_snapshot(rt, team_id, agent): - """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider - scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" - try: - snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} - e = snap.get(_snap_scope_key(team_id, agent)) - if isinstance(e, dict) and isinstance(e.get("rows"), list): - return float(e.get("ts") or 0), e["rows"] - except Exception: - pass - return None - - -def save_pool_snapshots(rt, taken_by=""): - """Persist every currently-cached pool scope as the pause-time snapshot ('the last - successful sync', made concrete). Ensures the consolidated default scope exists first so - a pause on a cold process still captures something to serve.""" - import time as _time - import routes_customers as _rc - try: - _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on - except Exception: - pass # cold + Odoo down: persist whatever IS cached - pools = {} - for key, entry in list(rt.pool_cache.items()): - if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" - and isinstance(entry, tuple) and len(entry) == 2 - and isinstance(entry[1], list)): - pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} - if not pools: - return 0 - - def _up(cur): - cur["odoo_pool"] = pools - cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") - cur["takenBy"] = str(taken_by or "") - return cur - - rt.update(SNAPSHOT_KEY, _up, flush="sync") - return len(pools) - - -def _rel_reconnect(rt): - """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot - disagree about what "resume" means.""" - import odoo_relational as rel - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = False - cur.pop("frozenAt", None) - cur.pop("frozenBy", None) - return cur - - rt.update(rel.CONFIG_KEY, _up, flush="sync") - return True - - -# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION -# -# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as -# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so -# the risk is what this block is mostly about. -# -# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env -# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already -# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, -# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build -# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks -# empty while its Odoo grids work. -# -# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns -# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — -# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under -# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector -# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That -# is D-259's exact shape (a pause written under one key and read under another) and -# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the -# SAME pass, and the carry is asserted. - - -def _env_odoo_fields(): - """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. - - ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. - Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a - PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo - offline, on a deployment where it had been working. The env fallback cannot save it, because the - entry's existence is what turns the fallback off. - ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the - field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two - vocabularies meet here; nowhere else. - """ - want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), - ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) - got = {field: (os.environ.get(env) or "").strip() for field, env in want} - missing = sorted(env for field, env in want if not got[field]) - if missing: - return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " - f"would take this tenant's Odoo offline rather than migrate it") - return got, "" - - -def migrate_env_odoo(rt): - """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. - - `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every - path including the skips, because "already migrated", "no keychain key on this deployment" and - "the env is incomplete" are three different operator actions. - - ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, - measured three times: a developer's CLI write to the tenant store is reverted by the running - Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success - every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here - would be a dry run that lies, and the thing it would lie about is a credential. - - ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a - third-party failure must not take the boot down. - """ - out = {"done": False, "entry": "", "carried_pause": False, "why": ""} - if not env_odoo_available(rt): - # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. - out["why"] = "this tenant has no environment Odoo credential to migrate" - return out - fields, why = _env_odoo_fields() - if not fields: - out["why"] = why - return out - # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the - # NEW key and the old one is unreachable through the resolver — so the only moment this fact can - # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. - try: - was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) - .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) - except Exception: # noqa: BLE001 - was_paused = False - row, why = _kc().ensure_entry_of_type( - rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") - if row is None: - out["why"] = why - return out - out["done"], out["entry"] = True, row["id"] - if was_paused: - # The freeze followed the credential. Without this the connector silently RESUMES pulling - # live Odoo at the first boot after the migration. - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - entry = dict(cur.get(row["id"]) or {}) - entry["paused"] = True - entry["pausedBy"] = "system" - entry["pausedNote"] = ("carried over from the environment source when the credential " - "was migrated onto the keychain") - cur[row["id"]] = entry - return cur - - try: - rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") - out["carried_pause"] = True - except Exception as exc: # noqa: BLE001 - # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse - # than one that did not run, so this is the one failure that must never be silent. - out["why"] = (f"the credential migrated but the PAUSE could not be carried over " - f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") - return out - - -def _own_row(session, entry_id): - """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the - caller cannot see: telling a member that somebody else's personal credential EXISTS is the - disclosure the scope is for.""" - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(entry_id)), None) - if row is None: - raise err(404, "no_entry", "no such key") - return row - - -@router.get("/admin/keychain") -def list_keychain(session: Session = Depends(require_session)): - """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. - R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would - ship the feature and no door to it. The wall moved INTO the payload: a member sees the - business-wide entries and their own, never anybody else's personal one.""" - kc = _kc() - return {"entries": visible_entries(session.runtime, session.uname, - bool(session.admin)), - "locked": not kc.unlocked(), - #: the vocabulary and the permission, so the client renders a picker it can honour - #: rather than offering an option the server will refuse (R4: business is admin-only). - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - "tenantWideTypes": list(TENANT_WIDE_TYPES)} - - -@router.post("/admin/keychain", status_code=201) -def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): - kc = _kc() - body = body or {} - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - scope = clean_scope(body.get("scope")) - if scope is None: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can create one. You can add it as a personal connection instead.") - etype = str(body.get("type") or "").strip().lower() - if scope == "personal" and etype in TENANT_WIDE_TYPES: - # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this - # credential is what the WHOLE workspace's databases are built from, so "personal" would - # be a label on a tenant-wide key rather than a boundary around it. - raise err(400, "scope_not_available", - f"a {etype} connection is what this whole workspace's databases are read " - f"through, so it is always business-wide — it cannot be a personal connection. " - f"An administrator can add it for everyone.") - try: - row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), - body.get("fields"), session.uname) - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", - f"the keychain is locked — {e}. A secret is never stored unencrypted.") - except ValueError as e: - raise err(400, "bad_entry", str(e)) - except Exception: - raise err(503, "store_unavailable", "the entry was not saved — try again") - # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode - # of a side bucket is to publish a credential, not to hide one. So the second write is not - # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was - # stored. `business` needs no row at all, so this branch is the only one that can be partial. - # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing - # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin - # find a second switch. The tables were never dropped, so "resume" is one flag. - if etype == "odoo": - try: - _rel_reconnect(session.runtime) - except Exception: # noqa: BLE001 - pass - if scope != DEFAULT_SCOPE: - try: - _write_scope(session.runtime, row["id"], scope, session.uname) - except Exception: - try: - kc.delete_entry(session.runtime, row["id"]) - except Exception: # noqa: BLE001 - pass - raise err(503, "store_unavailable", - "the key was not saved — its sharing setting could not be stored, so " - "nothing was kept. Try again.") - return {"entry": {**row, "scope": scope, "owner": session.uname}} - - -@router.put("/admin/keychain/{entry_id}") -def update_key(entry_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so - "edit this key" means "replace it" and that is `DELETE` + `POST`.""" - row = _own_row(session, entry_id) - scope = clean_scope((body or {}).get("scope"), default=None) - if scope is None: - raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can make one business-wide.") - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to change") - if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: - raise err(400, "scope_not_available", - f"a {row.get('type')} connection is what this whole workspace's databases are " - f"read through, so it is always business-wide.") - owner = row.get("owner") or session.uname - try: - _write_scope(session.runtime, entry_id, scope, owner) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} - - -@router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): - row = _own_row(session, entry_id) - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to delete") - try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} - - -@router.post("/admin/keychain/{entry_id}/test") -def test_key(entry_id: str, session: Session = Depends(require_session)): - _own_row(session, entry_id) # 404 for an entry this caller may not see - return _kc().test_entry(session.runtime, entry_id) - - -def _unsynced_customer_records(session): - """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no - longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a - tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the - answer is `known: False`, never a fabricated zero.""" - try: - import core.table_store as table_store - bucket = session.runtime.get("customer_table_workspace") or {} - overlay_pids = {} - for uname, ws in bucket.items(): - if uname == table_store.SHARED_KEY or not isinstance(ws, dict): - continue - for pid, cells in (ws.get("overlays") or {}).items(): - if isinstance(cells, dict) and cells: - overlay_pids.setdefault(str(pid), cells) - if not overlay_pids: - return {"known": True, "count": 0, "rows": []} - from routes_customers import allowed_pids - pool = {str(p) for p in allowed_pids(session)} - orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) - if str(x).isdigit() else 0) - rows = [] - for p in orphans[:50]: - cells = overlay_pids[p] - hint = next((str(v) for v in cells.values() if str(v).strip()), "") - rows.append({"pid": int(p) if str(p).isdigit() else p, - "fields": len(cells), "hint": hint[:80]}) - return {"known": True, "count": len(orphans), "rows": rows, - "shown": min(len(orphans), 50)} - except Exception as e: - return {"known": False, "count": None, "rows": [], - "note": f"pool unavailable — {type(e).__name__}"} - - -@router.get("/admin/connectors") -def connectors(session: Session = Depends(require_session)): - kc = _kc() - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same - # facts with a status column. Reading `kc.list_entries` here instead would have shown a member - # every colleague's personal connection on the screen next door to the one that hides them. - entries = visible_entries(session.runtime, session.uname, bool(session.admin)) - # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — - # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env - # for tenant #0 only, else nothing (fail closed — never another tenant's environment). - # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source - # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would - # make the answer depend on who opened the pane. Personal entries are excluded for the same - # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. - _scopes = _scope_rows(session.runtime) - first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) - if e["type"] == "odoo" - and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) - # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the - # connectors DIRECTORY answer "does the environment serve this tenant?" with one function - # instead of two spellings that agreed until a second tenant signed in. - if first_odoo and kc.unlocked(): - resolved = f"keychain:{first_odoo}" - elif env_odoo_available(session.runtime): - resolved = "env" - else: - resolved = None - rows = [] - if env_odoo_available(session.runtime): - rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", - "source": "env", "active": resolved == "env", - # the deployment's own credential — business-wide by construction, and it - # has no owner to be personal to. - "scope": DEFAULT_SCOPE, "owner": "", - "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) - for e in entries: - rows.append({"key": e["id"], "label": e["label"], "type": e["type"], - "source": "keychain", "preview": e["preview"], - "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", - "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), - "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) - out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. - # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the - # hole it left: "measures not already computed may still reach the source". D-10 - # closed that hole — `harness/datastore.py` (the mirror every measure column is - # answered from) refuses to sync while paused, and `routes_products._pool_for` got the - # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather - # than left standing, because a warning that outlives its defect teaches the reader to - # ignore warnings. - # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely - # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and - # collapsing them into "everything freezes" would be the kind of tidy summary that - # stops being true the first time one of them changes. - # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the - # correction is recorded here rather than silently applied. D-62 said this note - # "promises a behaviour on a dashboard measure path that has been dead since W16". - # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from - # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while - # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is - # `/api/v1/pages/{key}` (D-52), which this note never mentioned. - # - # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures - # stop moving rather than going blank". BOTH pool paths answer **503** when they have - # no copy to serve — the customer path for a scope with no snapshot - # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because - # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says - # so in as many words). So a paused connector plus a restarted server is exactly the - # blank screen this sentence promised could not happen. A warning that over-promises - # is worse than none: it is the sentence somebody quotes when the screen disagrees. - "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " - "views stay, and nothing reaches the source while it is paused. " - "Anything this server has already read keeps showing: the customer " - "workspace serves its pause-time snapshot, the product list serves " - "the last copy read since startup, and measure columns keep answering " - "from the mirror as it stood when you paused. What has NOT been read " - "cannot be shown — a scope with no snapshot, or the product list after " - "a restart, reports that the source is paused instead of showing " - "figures. Resume to start reading live again.")} - # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to - # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on - # the payload that is NOT about connectors: it unions overlays across EVERY user of the - # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user - # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member - # would have read colleagues' typed notes off the Connectors pane. It also filters against - # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and - # the number itself becomes wrong for them as well as private. - # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already - # returned, and the audit has to walk the payload, not the entry list I was thinking about. - if session.tenant == "royal-imports" and session.admin: - out["unsynced"] = _unsynced_customer_records(session) - return out - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database -# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to -# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a -# switch flipped here is a switch the sync path obeys — a config the route knows and the sync -# path does not is a control that does nothing and reports success. -def _rel(): - import odoo_relational as rel - return rel - - -def _odoo_entry(session): - """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing - is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" - kc = _kc() - scopes = _scope_rows(session.runtime) - for e in kc.list_entries(session.runtime): - if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": - return e - return None - - -def _odoo_source_fields(session, entry): - """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. - - ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no - route returns its output; this returns the three fields that identify WHICH server, and the - api key is not among them. The masked preview is the entry's own and was computed at write. - ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by - the process, and an admin editing it from a tenant screen would be editing the container. - """ - if entry is None: - return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), - os.environ.get("ODOO_USER", ""), False) - try: - f = _kc().read_fields(session.runtime, entry["id"]) or {} - except Exception: # noqa: BLE001 - return ("", "", "", True) # locked keychain: honest blanks, still editable - return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) - - -def _odoo_admin(session): - """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the - whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide - type cannot be personal in the first place.""" - if not session.admin: - raise err(403, "not_admin", - "the Odoo connection serves this whole workspace, so only an administrator can " - "configure it") - - -@router.get("/admin/connectors/odoo/config") -def odoo_config(session: Session = Depends(require_session)): - """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a - person needs to see before changing any of it.""" - _odoo_admin(session) - rel = _rel() - entry = _odoo_entry(session) - server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) - cfg = rel.read_config(session.runtime) - return { - "applicable": bool(rel.is_royal(session.tenant)), - "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) - else "none"), - "entryId": (entry or {}).get("id", ""), - "label": (entry or {}).get("label", "Odoo (environment)"), - "preview": (entry or {}).get("preview", ""), - "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, - "serverDbEditable": editable, - "grids": rel.grid_choices(session.runtime), - "syncEvery": cfg["syncEvery"], - "syncOptions": list(rel.SYNC_PRESETS), - "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, - "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], - # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: - # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders - # no button rather than one that 400s. - "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) - and not cfg["frozen"]), - } - - -@router.put("/admin/connectors/odoo/config") -def odoo_config_put(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED - back rather than silently applied.""" - _odoo_admin(session) - rel = _rel() - body = body or {} - notes = [] - - grids = body.get("grids") - known = {c["key"] for c in rel.grid_choices(session.runtime)} - clean_grids = None - if isinstance(grids, dict): - unknown = sorted(str(k) for k in grids if str(k) not in known) - if unknown: - # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid - # this connector does not have, and swallowing it makes the two disagree quietly. - raise err(400, "unknown_grid", - f"this connector has no grid called {', '.join(unknown)}") - clean_grids = {str(k): bool(v) for k, v in grids.items()} - if clean_grids and not any(clean_grids.get(k, True) for k in known): - notes.append("every grid is switched off — nothing will be materialised on the next " - "sync, and the databases you already have are left untouched") - - every = body.get("syncEvery") - clean_every = None - if every is not None: - clean_every = str(every).strip().lower() - if clean_every not in rel.SYNC_PRESETS: - # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. - # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, - # and never silently ignored either, because a control that discards your answer - # without a word is how a limit becomes invisible. - clean_every = rel.DEFAULT_SYNC - notes.append(f"{every!r} is not an interval this connector offers, and anything under " - f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " - f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") - - server_db = body.get("serverDb") - if server_db is not None: - server_db = " ".join(str(server_db).split())[:80] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if clean_grids is not None: - cur.setdefault("grids", {}).update(clean_grids) - if clean_every is not None: - cur["syncEvery"] = clean_every - return cur - - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - - if server_db: - notes.append(_rewrite_server_db(session, server_db)) - - # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE - # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: - # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( - # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So - # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value - # is stored, clamped, displayed and never read. - # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a - # default 1800 s and syncs anyway. "Only when I ask" asks all the same. - # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not - # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it - # is REPORTED here, at the moment of the change, with what it really controls. Delete these - # notes when the loop becomes per-tenant, not before. - if clean_every is not None: - if not rel.is_royal(session.tenant): - notes.append("this interval is saved, but the sync loop currently reads its schedule " - "from one workspace for the whole deployment — so it will not change how " - "often YOUR data refreshes until per-workspace scheduling ships") - else: - notes.append("this interval is saved and it is the one the deployment's sync loop " - "uses — it changes the refresh rate for every workspace on this " - "deployment, not only this one") - if clean_every == "manual": - notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " - "manual-only branch, so data still refreshes on the default interval") - - out = odoo_config(session) - return {**out, "notes": [n for n in notes if n]} - - -def _rewrite_server_db(session, server_db): - """Point the stored Odoo credential at a different server database (R9's first reading). - - ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of - how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. - So this is add-then-delete through the module's own doors, with the side rows (pause flag, - scope) carried across because they are keyed by ENTRY ID. - ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the - delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a - resync landing inside that window could read the OLD database. The alternative order can - leave the workspace with no credential at all, which is worse than one stale read. Milliseconds - of ambiguity beats a lost key. - """ - kc = _kc() - entry = _odoo_entry(session) - if entry is None: - return ("the server database is set on this deployment's environment, not in the " - "keychain, so it was not changed here") - try: - fields = kc.read_fields(session.runtime, entry["id"]) or {} - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", f"the keychain is locked — {e}") - if not fields: - raise err(400, "bad_entry", "this credential could not be read back to be changed") - if str(fields.get("db") or "") == server_db: - return "" - fields["db"] = server_db - try: - new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) - except Exception: - raise err(503, "store_unavailable", - "the server database was not changed — the existing connection is untouched") - # carry the side rows across, then retire the old entry - try: - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - if entry["id"] in flags: - def _mv(cur): - cur[new["id"]] = cur.pop(entry["id"], {}) - return cur - session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) - kc.delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - except Exception: # noqa: BLE001 - return (f"the connection now points at {server_db}, but the previous credential could " - f"not be removed — delete it under Keychains") - return f"the connection now points at the {server_db} database" - - -@router.post("/admin/connectors/odoo/disconnect") -def odoo_disconnect(session: Session = Depends(require_session)): - """R10 — remove the credential and FREEZE the grids as static data. - - ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the - key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — - including the boot rebuild and the resync loop, which for tenant #0 would otherwise - re-materialise from the process ENVIRONMENT and quietly undo the disconnect. - ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and - every FIELD DEFINITION stays, user-added columns included, because a field a person added is - the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes - one flag in a different bucket, which is what makes that guarantee structural rather than - careful. - """ - _odoo_admin(session) - rel = _rel() - import datetime as _dt - entry = _odoo_entry(session) - if not entry and not env_odoo_available(session.runtime): - raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = True - cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - cur["frozenBy"] = str(session.uname) - return cur - - # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT - # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which - # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, - # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read - # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, - # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected - # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's - # own sentence promised *"nothing is being refreshed"*. - # - # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause - # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` - # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed - # from `pause_connector` because it learned them the hard way: - # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no - # resolved source left to name, and the flag would land under a key nothing reads (D-10). - # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to - # serve; a failed capture leaves the connector live rather than paused-with-nothing. - _, flag_key = _resolved_odoo_key(session.runtime) - snapshots = 0 - if flag_key: - try: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - except Exception: # noqa: BLE001 - # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. - snapshots = 0 - - # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and - # the connector is still live; if the delete failed AFTER the flag landed, the tenant is - # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order - # can leave a workspace with no key and a connector that still tries to sync. - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "nothing was disconnected — try again") - - paused_mirror = False - if flag_key: - def _pause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": True} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") - paused_mirror = True - except Exception: # noqa: BLE001 - paused_mirror = False - - removed = "" - if entry is not None: - try: - _kc().delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - removed = entry["id"] - except Exception: - raise err(503, "store_unavailable", - "the databases are frozen but the stored credential was not removed — " - "delete it under Keychains") - - # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the - # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT - # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a - # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: - # the limit that cannot be removed is REPORTED, with what to do instead. - note = ("Your Odoo databases are frozen: every row and every column you had is still there and " - "still readable, and nothing is being refreshed.") - if not paused_mirror and flag_key: - note += (" ⚠ The live mirror could not be paused, so the two read-through databases " - "(order lines and GL lines) may keep advancing — pause the Odoo connector under " - "Keychains to stop them.") - note += (" Reconnecting adds the key back and resumes into the same databases." if entry - else " This workspace's Odoo credential comes from the deployment environment, so " - "there was no stored key to remove — the databases are frozen and Reconnect " - "resumes them into the same tables.") - return {"frozen": True, "removedEntry": removed, - # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean - # nobody returns is a guarantee nobody can check. - "pausedMirror": paused_mirror, "snapshots": snapshots, - "source": "keychain" if entry else "env", - "note": note} - - -@router.post("/admin/connectors/odoo/reconnect") -def odoo_reconnect(session: Session = Depends(require_session)): - """R10's second sentence — *"Reconnecting resumes into the same tables."* - - It clears the freeze and nothing else: the tables were never dropped, so there is nothing to - recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this - is the switch that lets the sync path see it again. - """ - _odoo_admin(session) - try: - _rel_reconnect(session.runtime) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only - # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the - # two read-through grids would sit at the disconnect date while the panel said "connected - # again" — the same disagreement between the halves of "frozen", pointing the other way. - # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the - # resolved source only exists again by this point. - _, flag_key = _resolved_odoo_key(session.runtime) - resumed = False - if flag_key: - def _unpause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": False} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") - resumed = True - except Exception: # noqa: BLE001 - resumed = False - connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) - return {"frozen": False, "connected": connected, "resumedMirror": resumed, - "note": ("Odoo is connected again and the databases you already had will refresh in " - "place." if connected else - "The freeze is lifted, but there is no Odoo credential yet — add one under " - "Keychains and the databases resume into the same tables.")} - - -@router.post("/admin/connectors/{key}/pause") -def pause_connector(key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - paused = bool((body or {}).get("paused")) - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an - # admin act; pausing your own personal one is yours. The env source has no keychain row and - # is business-wide by construction, hence the admin fallthrough. - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(key)), None) - if row is not None: - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to pause") - elif not session.admin: - raise err(403, "forbidden", "administrators only") - - # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a - # "last successful sync" to serve before the freeze takes effect. Capturing before the - # flag flips means a failed capture leaves the connector live (never paused-with-nothing). - snapshots = 0 - _, flag_key = _resolved_odoo_key(session.runtime) - if paused and flag_key and str(key) == flag_key: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - - def _up(cur): - cur[str(key)] = {"paused": paused} - return cur - - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"key": key, "paused": paused, "snapshots": snapshots} +"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). + +Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER +return a decrypted field — list rows carry a masked preview, and the decrypt function is a +connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's +env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** +count (rows holding overlay data whose pids the current pool no longer serves; counted and +drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly +in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with +the keychain cutover wave R3 staged. +""" +import os + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session +# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door +# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall +# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per +# entry instead of per endpoint. `session.admin` is still what business-wide requires. + +router = APIRouter(prefix="/api/v1") + +#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by +#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another +#: freezes nothing while reporting success, which is the shape of the bug D-10 books. +from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 + ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) +#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED +#: odoo connector is paused; read by the pool path while paused; survives a Space restart. +SNAPSHOT_KEY = "connector_snapshots" + +#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. +#: +#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves +#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and +#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's +#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's +#: connection, and handing it to another tenant is the leak this method exists to prevent."* +#: +#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and +#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value +#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, +#: the gate reds instead of the product quietly disagreeing with itself. +ENV_ODOO_TENANT = "royal-imports" + + +def env_odoo_available(rt): + """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) + + ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. + `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant + guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was + `connected` and offered "Manage keys" for a credential belonging to another company. This + module asked the same question correctly two functions below, which is the whole shape of + [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in + as the second tenant. + + ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which + source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, + wrong for "should the environment row be listed at all", which is what the connectors pane + needs in order to show an inactive env source beside an active keychain one. + + Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is + how another tenant's environment got reported as this tenant's connection in the first place. + """ + try: + if not (os.environ.get("ODOO_URL") or "").strip(): + return False + return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT + except Exception: # noqa: BLE001 + return False + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. +# Every keychain entry and connector carries a scope, selectable per connection; business-wide is +# admin-only and applies to every user in the tenant."* +# +# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file +# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is +# an access rule, not a secret, so it lives in the layer that already decides who may ask. +# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves +# cannot drift into two vocabularies. +SCOPES = ("business", "personal") +#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before +#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. +#: Materialising it would be a migration nobody authorised and would rewrite the store on the next +#: read [[a-migration-that-runs-on-the-next-write]]. +DEFAULT_SCOPE = "business" +#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. +#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean +#: the same thing and there is only one way to spell the common case. +SCOPE_KEY = "keychain_scopes" + +#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. +#: +#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* +#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is +#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they +#: would silently become the credential the entire workspace's databases are built from, which is +#: a credential elevation wearing a scope picker. +#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes +#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is +#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door +#: from the only side B owns, and the resolver-side guard is booked for A. +TENANT_WIDE_TYPES = ("odoo", "meta_ads") + + +def clean_scope(raw, default=DEFAULT_SCOPE): + """A scope word from the wire, or None when the caller said something we do not speak. + + Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole + reason this returns None rather than falling back: a typo'd `"personel"` silently becoming + business-wide is exactly the failure a scope picker exists to prevent. + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + got = str(raw).strip().lower() + return got if got in SCOPES else None + + +def _scope_rows(rt): + try: + return dict(rt.get(SCOPE_KEY) or {}) + except Exception: # noqa: BLE001 + return {} + + +def entry_scope(rt, entry_id, rows=None): + """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a + census does not re-read the store once per entry.""" + r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) + if not isinstance(r, dict): + return DEFAULT_SCOPE, "" + return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) + + +def may_see(scope, owner, uname): + """R4's visibility rule: business-wide is everyone's, personal is its owner's. + + ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the + per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token + is identity, not infrastructure"*). An admin who could read every member's personal credential + would make "personal" a label rather than a boundary. + """ + return scope != "personal" or str(owner) == str(uname) + + +def visible_entries(rt, uname, is_admin=False): + """This USER's view of the keychain: every business entry plus their own personal ones, each + row carrying its `scope` and `owner` so no caller has to ask a second time. + + ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can + hold a connection of their own; it does not hand them four characters of the workspace's Odoo + key. So a business row a member did not create arrives WITHOUT `preview` — they can see that + the connection exists and is theirs to use, which is the whole of what R4 grants. The default + is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. + """ + rows = _scope_rows(rt) + out = [] + for e in _kc().list_entries(rt): + scope, owner = entry_scope(rt, e["id"], rows) + if not owner: + owner = str(e.get("createdBy") or "") + if not may_see(scope, owner, uname): + continue + row = {**e, "scope": scope, "owner": owner} + if not (is_admin or str(owner) == str(uname)): + row["preview"] = "" + out.append(row) + return out + + +def _write_scope(rt, entry_id, scope, owner): + """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the + default, so the store holds one spelling of the common case.""" + def _up(cur): + if scope == DEFAULT_SCOPE: + cur.pop(str(entry_id), None) + else: + cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} + return cur + + rt.update(SCOPE_KEY, _up, flush="sync") + return True + + +def _may_touch(session, row): + """May this session change or delete `row`? An admin owns the business-wide ones; a member + owns their own personal ones. Anything else is not theirs to move.""" + if row.get("scope") == "personal": + return str(row.get("owner") or "") == str(session.uname) + return bool(session.admin) + + +def _kc(): + import core.keychain as keychain + return keychain + + +def _resolved_odoo_key(rt): + """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the + shapes this module needs: the display string (`env` / `keychain:`) and the key the pause + flag is stored under. + + ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, + beside the `odoo_source()` it must agree with. This function had its own copy of the same + three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and + a second copy is exactly how a pause flag comes to be written against one resolution and read + against another, freezing nothing while the UI reports success. The two SHAPES stay here + because they are this module's presentation concern; the DECISION does not. + """ + flag_key = rt.odoo_flag_key() + if not flag_key: + return None, None + return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key + + +def odoo_paused(rt): + """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry + that is not the resolved source freezes nothing — it serves nothing. + + ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the + measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question + the customer pool does. This name stays because `routes_customers`, `routes_products` and + `verify_api` all call it — moving the logic without moving the door keeps one answer and + costs no caller a change. + """ + return bool(rt.odoo_paused()) + + +def _snap_scope_key(team_id, agent): + return f"t={team_id}|a={agent}" + + +def load_pool_snapshot(rt, team_id, agent): + """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider + scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" + try: + snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} + e = snap.get(_snap_scope_key(team_id, agent)) + if isinstance(e, dict) and isinstance(e.get("rows"), list): + return float(e.get("ts") or 0), e["rows"] + except Exception: + pass + return None + + +def save_pool_snapshots(rt, taken_by=""): + """Persist every currently-cached pool scope as the pause-time snapshot ('the last + successful sync', made concrete). Ensures the consolidated default scope exists first so + a pause on a cold process still captures something to serve.""" + import time as _time + import routes_customers as _rc + try: + _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on + except Exception: + pass # cold + Odoo down: persist whatever IS cached + pools = {} + for key, entry in list(rt.pool_cache.items()): + if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" + and isinstance(entry, tuple) and len(entry) == 2 + and isinstance(entry[1], list)): + pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} + if not pools: + return 0 + + def _up(cur): + cur["odoo_pool"] = pools + cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") + cur["takenBy"] = str(taken_by or "") + return cur + + rt.update(SNAPSHOT_KEY, _up, flush="sync") + return len(pools) + + +def _rel_reconnect(rt): + """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot + disagree about what "resume" means.""" + import odoo_relational as rel + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = False + cur.pop("frozenAt", None) + cur.pop("frozenBy", None) + return cur + + rt.update(rel.CONFIG_KEY, _up, flush="sync") + return True + + +# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION +# +# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as +# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so +# the risk is what this block is mostly about. +# +# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env +# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already +# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, +# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build +# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks +# empty while its Odoo grids work. +# +# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns +# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — +# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under +# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector +# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That +# is D-259's exact shape (a pause written under one key and read under another) and +# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the +# SAME pass, and the carry is asserted. + + +def _env_odoo_fields(): + """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. + + ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. + Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a + PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo + offline, on a deployment where it had been working. The env fallback cannot save it, because the + entry's existence is what turns the fallback off. + ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the + field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two + vocabularies meet here; nowhere else. + """ + want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), + ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) + got = {field: (os.environ.get(env) or "").strip() for field, env in want} + missing = sorted(env for field, env in want if not got[field]) + if missing: + return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " + f"would take this tenant's Odoo offline rather than migrate it") + return got, "" + + +def migrate_env_odoo(rt): + """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. + + `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every + path including the skips, because "already migrated", "no keychain key on this deployment" and + "the env is incomplete" are three different operator actions. + + ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success + every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here + would be a dry run that lies, and the thing it would lie about is a credential. + + ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a + third-party failure must not take the boot down. + """ + out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + if not env_odoo_available(rt): + # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. + out["why"] = "this tenant has no environment Odoo credential to migrate" + return out + fields, why = _env_odoo_fields() + if not fields: + out["why"] = why + return out + # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the + # NEW key and the old one is unreachable through the resolver — so the only moment this fact can + # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. + try: + was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) + .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) + except Exception: # noqa: BLE001 + was_paused = False + row, why = _kc().ensure_entry_of_type( + rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") + if row is None: + out["why"] = why + return out + out["done"], out["entry"] = True, row["id"] + if was_paused: + # The freeze followed the credential. Without this the connector silently RESUMES pulling + # live Odoo at the first boot after the migration. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = dict(cur.get(row["id"]) or {}) + entry["paused"] = True + entry["pausedBy"] = "system" + entry["pausedNote"] = ("carried over from the environment source when the credential " + "was migrated onto the keychain") + cur[row["id"]] = entry + return cur + + try: + rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + out["carried_pause"] = True + except Exception as exc: # noqa: BLE001 + # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse + # than one that did not run, so this is the one failure that must never be silent. + out["why"] = (f"the credential migrated but the PAUSE could not be carried over " + f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + return out + + +def _own_row(session, entry_id): + """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the + caller cannot see: telling a member that somebody else's personal credential EXISTS is the + disclosure the scope is for.""" + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(entry_id)), None) + if row is None: + raise err(404, "no_entry", "no such key") + return row + + +@router.get("/admin/keychain") +def list_keychain(session: Session = Depends(require_session)): + """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. + R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would + ship the feature and no door to it. The wall moved INTO the payload: a member sees the + business-wide entries and their own, never anybody else's personal one.""" + kc = _kc() + return {"entries": visible_entries(session.runtime, session.uname, + bool(session.admin)), + "locked": not kc.unlocked(), + #: the vocabulary and the permission, so the client renders a picker it can honour + #: rather than offering an option the server will refuse (R4: business is admin-only). + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + "tenantWideTypes": list(TENANT_WIDE_TYPES)} + + +@router.post("/admin/keychain", status_code=201) +def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): + kc = _kc() + body = body or {} + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + scope = clean_scope(body.get("scope")) + if scope is None: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can create one. You can add it as a personal connection instead.") + etype = str(body.get("type") or "").strip().lower() + if scope == "personal" and etype in TENANT_WIDE_TYPES: + # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this + # credential is what the WHOLE workspace's databases are built from, so "personal" would + # be a label on a tenant-wide key rather than a boundary around it. + raise err(400, "scope_not_available", + f"a {etype} connection is what this whole workspace's databases are read " + f"through, so it is always business-wide — it cannot be a personal connection. " + f"An administrator can add it for everyone.") + try: + row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), + body.get("fields"), session.uname) + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", + f"the keychain is locked — {e}. A secret is never stored unencrypted.") + except ValueError as e: + raise err(400, "bad_entry", str(e)) + except Exception: + raise err(503, "store_unavailable", "the entry was not saved — try again") + # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode + # of a side bucket is to publish a credential, not to hide one. So the second write is not + # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was + # stored. `business` needs no row at all, so this branch is the only one that can be partial. + # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing + # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin + # find a second switch. The tables were never dropped, so "resume" is one flag. + if etype == "odoo": + try: + _rel_reconnect(session.runtime) + except Exception: # noqa: BLE001 + pass + if scope != DEFAULT_SCOPE: + try: + _write_scope(session.runtime, row["id"], scope, session.uname) + except Exception: + try: + kc.delete_entry(session.runtime, row["id"]) + except Exception: # noqa: BLE001 + pass + raise err(503, "store_unavailable", + "the key was not saved — its sharing setting could not be stored, so " + "nothing was kept. Try again.") + return {"entry": {**row, "scope": scope, "owner": session.uname}} + + +@router.put("/admin/keychain/{entry_id}") +def update_key(entry_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so + "edit this key" means "replace it" and that is `DELETE` + `POST`.""" + row = _own_row(session, entry_id) + scope = clean_scope((body or {}).get("scope"), default=None) + if scope is None: + raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can make one business-wide.") + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to change") + if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: + raise err(400, "scope_not_available", + f"a {row.get('type')} connection is what this whole workspace's databases are " + f"read through, so it is always business-wide.") + owner = row.get("owner") or session.uname + try: + _write_scope(session.runtime, entry_id, scope, owner) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} + + +@router.delete("/admin/keychain/{entry_id}") +def delete_key(entry_id: str, session: Session = Depends(require_session)): + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + try: + _kc().delete_entry(session.runtime, entry_id) + _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + return {"ok": True} + + +@router.post("/admin/keychain/{entry_id}/test") +def test_key(entry_id: str, session: Session = Depends(require_session)): + _own_row(session, entry_id) # 404 for an entry this caller may not see + return _kc().test_entry(session.runtime, entry_id) + + +def _unsynced_customer_records(session): + """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no + longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a + tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the + answer is `known: False`, never a fabricated zero.""" + try: + import core.table_store as table_store + bucket = session.runtime.get("customer_table_workspace") or {} + overlay_pids = {} + for uname, ws in bucket.items(): + if uname == table_store.SHARED_KEY or not isinstance(ws, dict): + continue + for pid, cells in (ws.get("overlays") or {}).items(): + if isinstance(cells, dict) and cells: + overlay_pids.setdefault(str(pid), cells) + if not overlay_pids: + return {"known": True, "count": 0, "rows": []} + from routes_customers import allowed_pids + pool = {str(p) for p in allowed_pids(session)} + orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) + if str(x).isdigit() else 0) + rows = [] + for p in orphans[:50]: + cells = overlay_pids[p] + hint = next((str(v) for v in cells.values() if str(v).strip()), "") + rows.append({"pid": int(p) if str(p).isdigit() else p, + "fields": len(cells), "hint": hint[:80]}) + return {"known": True, "count": len(orphans), "rows": rows, + "shown": min(len(orphans), 50)} + except Exception as e: + return {"known": False, "count": None, "rows": [], + "note": f"pool unavailable — {type(e).__name__}"} + + +@router.get("/admin/connectors") +def connectors(session: Session = Depends(require_session)): + kc = _kc() + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same + # facts with a status column. Reading `kc.list_entries` here instead would have shown a member + # every colleague's personal connection on the screen next door to the one that hides them. + entries = visible_entries(session.runtime, session.uname, bool(session.admin)) + # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — + # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env + # for tenant #0 only, else nothing (fail closed — never another tenant's environment). + # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source + # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would + # make the answer depend on who opened the pane. Personal entries are excluded for the same + # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. + _scopes = _scope_rows(session.runtime) + first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) + if e["type"] == "odoo" + and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) + # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the + # connectors DIRECTORY answer "does the environment serve this tenant?" with one function + # instead of two spellings that agreed until a second tenant signed in. + if first_odoo and kc.unlocked(): + resolved = f"keychain:{first_odoo}" + elif env_odoo_available(session.runtime): + resolved = "env" + else: + resolved = None + rows = [] + if env_odoo_available(session.runtime): + rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", + "source": "env", "active": resolved == "env", + # the deployment's own credential — business-wide by construction, and it + # has no owner to be personal to. + "scope": DEFAULT_SCOPE, "owner": "", + "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) + for e in entries: + rows.append({"key": e["id"], "label": e["label"], "type": e["type"], + "source": "keychain", "preview": e["preview"], + "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", + "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), + "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) + out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. + # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the + # hole it left: "measures not already computed may still reach the source". D-10 + # closed that hole — `harness/datastore.py` (the mirror every measure column is + # answered from) refuses to sync while paused, and `routes_products._pool_for` got the + # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather + # than left standing, because a warning that outlives its defect teaches the reader to + # ignore warnings. + # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely + # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and + # collapsing them into "everything freezes" would be the kind of tidy summary that + # stops being true the first time one of them changes. + # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the + # correction is recorded here rather than silently applied. D-62 said this note + # "promises a behaviour on a dashboard measure path that has been dead since W16". + # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from + # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while + # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is + # `/api/v1/pages/{key}` (D-52), which this note never mentioned. + # + # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures + # stop moving rather than going blank". BOTH pool paths answer **503** when they have + # no copy to serve — the customer path for a scope with no snapshot + # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because + # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says + # so in as many words). So a paused connector plus a restarted server is exactly the + # blank screen this sentence promised could not happen. A warning that over-promises + # is worse than none: it is the sentence somebody quotes when the screen disagrees. + "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " + "views stay, and nothing reaches the source while it is paused. " + "Anything this server has already read keeps showing: the customer " + "workspace serves its pause-time snapshot, the product list serves " + "the last copy read since startup, and measure columns keep answering " + "from the mirror as it stood when you paused. What has NOT been read " + "cannot be shown — a scope with no snapshot, or the product list after " + "a restart, reports that the source is paused instead of showing " + "figures. Resume to start reading live again.")} + # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to + # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on + # the payload that is NOT about connectors: it unions overlays across EVERY user of the + # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user + # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member + # would have read colleagues' typed notes off the Connectors pane. It also filters against + # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and + # the number itself becomes wrong for them as well as private. + # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already + # returned, and the audit has to walk the payload, not the entry list I was thinking about. + if session.tenant == "royal-imports" and session.admin: + out["unsynced"] = _unsynced_customer_records(session) + return out + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database +# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to +# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a +# switch flipped here is a switch the sync path obeys — a config the route knows and the sync +# path does not is a control that does nothing and reports success. +def _rel(): + import odoo_relational as rel + return rel + + +def _odoo_entry(session): + """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing + is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" + kc = _kc() + scopes = _scope_rows(session.runtime) + for e in kc.list_entries(session.runtime): + if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": + return e + return None + + +def _odoo_source_fields(session, entry): + """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. + + ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no + route returns its output; this returns the three fields that identify WHICH server, and the + api key is not among them. The masked preview is the entry's own and was computed at write. + ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by + the process, and an admin editing it from a tenant screen would be editing the container. + """ + if entry is None: + return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), + os.environ.get("ODOO_USER", ""), False) + try: + f = _kc().read_fields(session.runtime, entry["id"]) or {} + except Exception: # noqa: BLE001 + return ("", "", "", True) # locked keychain: honest blanks, still editable + return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) + + +def _odoo_admin(session): + """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the + whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide + type cannot be personal in the first place.""" + if not session.admin: + raise err(403, "not_admin", + "the Odoo connection serves this whole workspace, so only an administrator can " + "configure it") + + +@router.get("/admin/connectors/odoo/config") +def odoo_config(session: Session = Depends(require_session)): + """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a + person needs to see before changing any of it.""" + _odoo_admin(session) + rel = _rel() + entry = _odoo_entry(session) + server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) + cfg = rel.read_config(session.runtime) + return { + "applicable": bool(rel.is_royal(session.tenant)), + "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) + else "none"), + "entryId": (entry or {}).get("id", ""), + "label": (entry or {}).get("label", "Odoo (environment)"), + "preview": (entry or {}).get("preview", ""), + "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, + "serverDbEditable": editable, + "grids": rel.grid_choices(session.runtime), + "syncEvery": cfg["syncEvery"], + "syncOptions": list(rel.SYNC_PRESETS), + "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, + "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], + # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: + # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders + # no button rather than one that 400s. + "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) + and not cfg["frozen"]), + } + + +@router.put("/admin/connectors/odoo/config") +def odoo_config_put(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED + back rather than silently applied.""" + _odoo_admin(session) + rel = _rel() + body = body or {} + notes = [] + + grids = body.get("grids") + known = {c["key"] for c in rel.grid_choices(session.runtime)} + clean_grids = None + if isinstance(grids, dict): + unknown = sorted(str(k) for k in grids if str(k) not in known) + if unknown: + # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid + # this connector does not have, and swallowing it makes the two disagree quietly. + raise err(400, "unknown_grid", + f"this connector has no grid called {', '.join(unknown)}") + clean_grids = {str(k): bool(v) for k, v in grids.items()} + if clean_grids and not any(clean_grids.get(k, True) for k in known): + notes.append("every grid is switched off — nothing will be materialised on the next " + "sync, and the databases you already have are left untouched") + + every = body.get("syncEvery") + clean_every = None + if every is not None: + clean_every = str(every).strip().lower() + if clean_every not in rel.SYNC_PRESETS: + # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. + # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, + # and never silently ignored either, because a control that discards your answer + # without a word is how a limit becomes invisible. + clean_every = rel.DEFAULT_SYNC + notes.append(f"{every!r} is not an interval this connector offers, and anything under " + f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " + f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") + + server_db = body.get("serverDb") + if server_db is not None: + server_db = " ".join(str(server_db).split())[:80] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if clean_grids is not None: + cur.setdefault("grids", {}).update(clean_grids) + if clean_every is not None: + cur["syncEvery"] = clean_every + return cur + + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + + if server_db: + notes.append(_rewrite_server_db(session, server_db)) + + # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE + # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: + # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( + # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So + # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value + # is stored, clamped, displayed and never read. + # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a + # default 1800 s and syncs anyway. "Only when I ask" asks all the same. + # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not + # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it + # is REPORTED here, at the moment of the change, with what it really controls. Delete these + # notes when the loop becomes per-tenant, not before. + if clean_every is not None: + if not rel.is_royal(session.tenant): + notes.append("this interval is saved, but the sync loop currently reads its schedule " + "from one workspace for the whole deployment — so it will not change how " + "often YOUR data refreshes until per-workspace scheduling ships") + else: + notes.append("this interval is saved and it is the one the deployment's sync loop " + "uses — it changes the refresh rate for every workspace on this " + "deployment, not only this one") + if clean_every == "manual": + notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " + "manual-only branch, so data still refreshes on the default interval") + + out = odoo_config(session) + return {**out, "notes": [n for n in notes if n]} + + +def _rewrite_server_db(session, server_db): + """Point the stored Odoo credential at a different server database (R9's first reading). + + ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of + how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. + So this is add-then-delete through the module's own doors, with the side rows (pause flag, + scope) carried across because they are keyed by ENTRY ID. + ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the + delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a + resync landing inside that window could read the OLD database. The alternative order can + leave the workspace with no credential at all, which is worse than one stale read. Milliseconds + of ambiguity beats a lost key. + """ + kc = _kc() + entry = _odoo_entry(session) + if entry is None: + return ("the server database is set on this deployment's environment, not in the " + "keychain, so it was not changed here") + try: + fields = kc.read_fields(session.runtime, entry["id"]) or {} + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", f"the keychain is locked — {e}") + if not fields: + raise err(400, "bad_entry", "this credential could not be read back to be changed") + if str(fields.get("db") or "") == server_db: + return "" + fields["db"] = server_db + try: + new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) + except Exception: + raise err(503, "store_unavailable", + "the server database was not changed — the existing connection is untouched") + # carry the side rows across, then retire the old entry + try: + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + if entry["id"] in flags: + def _mv(cur): + cur[new["id"]] = cur.pop(entry["id"], {}) + return cur + session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) + kc.delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + except Exception: # noqa: BLE001 + return (f"the connection now points at {server_db}, but the previous credential could " + f"not be removed — delete it under Keychains") + return f"the connection now points at the {server_db} database" + + +@router.post("/admin/connectors/odoo/disconnect") +def odoo_disconnect(session: Session = Depends(require_session)): + """R10 — remove the credential and FREEZE the grids as static data. + + ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the + key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — + including the boot rebuild and the resync loop, which for tenant #0 would otherwise + re-materialise from the process ENVIRONMENT and quietly undo the disconnect. + ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and + every FIELD DEFINITION stays, user-added columns included, because a field a person added is + the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes + one flag in a different bucket, which is what makes that guarantee structural rather than + careful. + """ + _odoo_admin(session) + rel = _rel() + import datetime as _dt + entry = _odoo_entry(session) + if not entry and not env_odoo_available(session.runtime): + raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = True + cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + cur["frozenBy"] = str(session.uname) + return cur + + # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT + # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which + # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, + # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read + # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, + # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected + # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's + # own sentence promised *"nothing is being refreshed"*. + # + # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause + # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` + # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed + # from `pause_connector` because it learned them the hard way: + # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no + # resolved source left to name, and the flag would land under a key nothing reads (D-10). + # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to + # serve; a failed capture leaves the connector live rather than paused-with-nothing. + _, flag_key = _resolved_odoo_key(session.runtime) + snapshots = 0 + if flag_key: + try: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + except Exception: # noqa: BLE001 + # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. + snapshots = 0 + + # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and + # the connector is still live; if the delete failed AFTER the flag landed, the tenant is + # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order + # can leave a workspace with no key and a connector that still tries to sync. + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "nothing was disconnected — try again") + + paused_mirror = False + if flag_key: + def _pause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": True} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") + paused_mirror = True + except Exception: # noqa: BLE001 + paused_mirror = False + + removed = "" + if entry is not None: + try: + _kc().delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + removed = entry["id"] + except Exception: + raise err(503, "store_unavailable", + "the databases are frozen but the stored credential was not removed — " + "delete it under Keychains") + + # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the + # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT + # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a + # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: + # the limit that cannot be removed is REPORTED, with what to do instead. + note = ("Your Odoo databases are frozen: every row and every column you had is still there and " + "still readable, and nothing is being refreshed.") + if not paused_mirror and flag_key: + note += (" ⚠ The live mirror could not be paused, so the two read-through databases " + "(order lines and GL lines) may keep advancing — pause the Odoo connector under " + "Keychains to stop them.") + note += (" Reconnecting adds the key back and resumes into the same databases." if entry + else " This workspace's Odoo credential comes from the deployment environment, so " + "there was no stored key to remove — the databases are frozen and Reconnect " + "resumes them into the same tables.") + return {"frozen": True, "removedEntry": removed, + # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean + # nobody returns is a guarantee nobody can check. + "pausedMirror": paused_mirror, "snapshots": snapshots, + "source": "keychain" if entry else "env", + "note": note} + + +@router.post("/admin/connectors/odoo/reconnect") +def odoo_reconnect(session: Session = Depends(require_session)): + """R10's second sentence — *"Reconnecting resumes into the same tables."* + + It clears the freeze and nothing else: the tables were never dropped, so there is nothing to + recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this + is the switch that lets the sync path see it again. + """ + _odoo_admin(session) + try: + _rel_reconnect(session.runtime) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only + # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the + # two read-through grids would sit at the disconnect date while the panel said "connected + # again" — the same disagreement between the halves of "frozen", pointing the other way. + # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the + # resolved source only exists again by this point. + _, flag_key = _resolved_odoo_key(session.runtime) + resumed = False + if flag_key: + def _unpause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": False} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") + resumed = True + except Exception: # noqa: BLE001 + resumed = False + connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) + return {"frozen": False, "connected": connected, "resumedMirror": resumed, + "note": ("Odoo is connected again and the databases you already had will refresh in " + "place." if connected else + "The freeze is lifted, but there is no Odoo credential yet — add one under " + "Keychains and the databases resume into the same tables.")} + + +@router.post("/admin/connectors/{key}/pause") +def pause_connector(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + paused = bool((body or {}).get("paused")) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an + # admin act; pausing your own personal one is yours. The env source has no keychain row and + # is business-wide by construction, hence the admin fallthrough. + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(key)), None) + if row is not None: + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to pause") + elif not session.admin: + raise err(403, "forbidden", "administrators only") + + # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a + # "last successful sync" to serve before the freeze takes effect. Capturing before the + # flag flips means a failed capture leaves the connector live (never paused-with-nothing). + snapshots = 0 + _, flag_key = _resolved_odoo_key(session.runtime) + if paused and flag_key and str(key) == flag_key: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + + def _up(cur): + cur[str(key)] = {"paused": paused} + return cur + + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"key": key, "paused": paused, "snapshots": snapshots} diff --git a/api/routes_nav.py b/api/routes_nav.py index bd154c7c64e4c805144f13cfa9fafc76ab1b0867..5d1f0d2d1dea2feb4809201c5a7944642db17620 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -1,769 +1,769 @@ -"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. - -SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who -may see what — a nav that hides a link the API would still serve is a UI courtesy, not a -permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), -so the nav and the 403 can never disagree about a grant. - -The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no -`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the -`prefs.json` Library preference is deliberately NOT applied — is documented on -`core.perms.nav_pages`, which is where it belongs: one place, both callers. -""" -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, perms, require_session - -router = APIRouter(prefix="/api/v1") - -#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder -#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a -#: surface (the server-filtered nav still decides what exists). -_NAV_PREFS_KEY = "nav_prefs" -_MAX_NAV_FOLDERS = 16 - -#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. -#: -#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a -#: separate bucket rather than another field in that one. Folders and placement are one -#: person's arrangement of their own rail — per-user by definition. What a database is CALLED -#: and what it looks like are facts about the database: a workspace where two people call the -#: same table different things has no shared vocabulary left to discuss it in. Same store, two -#: buckets, because they answer to two different owners. -#: -#: Shape: {"": {"icon": {"shape": , "tone": }, -#: "name": ""}} -_NAV_META_KEY = "nav_meta" - -#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these -#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, -#: so it is the one place the list is written twice. -#: -#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by -#: shape and maps the result, so a shape this whitelist let through and the renderer does not -#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the -#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of -#: that. If the two lists ever drift the symptom is an icon that silently reverts to the -#: default, which is the loudest SAFE failure available here. -_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", - "chart", "map", "users", "clock", "heart", "bolt"}) -_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) -_MAX_NAV_NAME = 60 - -#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. -#: -#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is -#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature -#: rather than a convenience one. -#: -#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. -#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An -#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one -#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls -#: off the list while forty slots hold repeat visits to the first. Keying by page makes -#: "recent" mean what the word means, and makes the cap bound the number of DATABASES -#: remembered rather than the number of clicks. -_NAV_RECENTS_KEY = "nav_recents" -_MAX_RECENTS = 50 - -#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a -#: precedent, not a preference. `user_tables.create` writes -#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses -#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 -#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — -#: with nothing to go red, because both ends are internally consistent. An integer instant has -#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect -#: shipped; this is that lesson applied before it.) -def _now() -> int: - return int(time.time()) - - -def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): - """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. - - `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown - or invisible key is dropped (it can return when the grant does — placement is cosmetic, - so pruning is loss-free). Unknown folder refs drop the placement, not the folder. - - `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the - `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than - pruned: keeping a placement whose database may since have been deleted is cosmetically - harmless, while pruning a live one is the silent data loss this function just stopped - causing. Never widened to non-`ut_` keys — those come from the compiled registry, which - cannot fail to enumerate. - """ - raw = raw if isinstance(raw, dict) else {} - folders, seen = [], set() - for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:40] - name = " ".join(str(f.get("name") or "").split())[:40] - if not fid or fid in seen or not name: - continue - seen.add(fid) - folders.append({"id": fid, "name": name}) - ids = {f["id"] for f in folders} - placement = {} - src = raw.get("placement") - if isinstance(src, dict): - for k, v in src.items(): - k, v = str(k)[:60], str(v)[:40] - if v not in ids: - continue # the folder is gone; the placement goes with it - if page_keys is None or k in page_keys: - placement[k] = v - elif keep_unknown_ut and k.startswith("ut_"): - placement[k] = v - return {"folders": folders, "placement": placement} - - -def _placeable_top_keys(session): - """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. - - ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by - construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and - `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's - user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. - So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement - naming one — on READ and on WRITE. - - The symptom was "a database I drag into a folder falls out of it later", never "it does not - work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy - (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. - The next reload served the pruned copy and the database was back at root. Built-in modules - (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly - why it read as intermittent rather than as a missing feature. - - ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents - block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the - ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 - fix did not reach. - - `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set - as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement - because the store blinked would be the original defect wearing a different cause. - """ - pages = perms.nav_pages(session.user) or [] - keys = {p["key"] for p in pages if not p.get("parent")} - try: - import core.user_tables as user_tables - # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never - # name a database this session cannot see, and the nav and this door agree by sharing - # one predicate rather than by two lists being kept in step. - # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is - # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires - # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured - # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the - # owner reports as "the Automation row arrives ten seconds late". - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - keys.add(str(e.get("key") or "")) - except Exception: - return keys, False - return keys, True - - -def _clean_icon(raw): - """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" - if not isinstance(raw, dict): - return None - shape, tone = raw.get("shape"), raw.get("tone") - if shape not in _ICON_SHAPES or tone not in _ICON_TONES: - return None - return {"shape": shape, "tone": tone} - - -def _read_nav_meta(runtime): - """The tenant's `nav_meta`, validated on the way OUT as well as in. - - Re-validating a read looks redundant and is not: the bucket outlives this code, an older - build may have written a shape this one no longer accepts, and the whitelist is mirrored - from a vocabulary that lives in another language. A row whose icon does not survive - validation renders the default mark — never a crash, and never a half-drawn glyph. - """ - try: - stored = runtime.get(_NAV_META_KEY) or {} - except Exception: - return {} # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return {} - out = {} - for key, meta in stored.items(): - if not isinstance(meta, dict): - continue - entry = {} - icon = _clean_icon(meta.get("icon")) - if icon: - entry["icon"] = icon - # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write - # route refuses a name for a built-in key, but a record written by an older build (or - # by hand) could still carry one — and honouring it would let the store rename - # `customer_data` on screen while `core/registry.py` and every other reader went on - # calling it Customer. A name that could not be written today is not served today. - name = meta.get("name") - if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): - entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] - if entry: - out[str(key)] = entry - return out - - -def _read_recents(runtime, uname, allowed): - """This user's recents, newest first, PRUNED to what they may currently see. - - ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: - - · the WRITE happens on every route open — it is the one hot path this file has — so it - must not build the whole nav to validate one key; - · a key whose grant was REVOKED must stop being offered without anyone running a - migration, and must come back if the grant does. That is `_clean_nav_prefs`' own - prune-never-invent posture, applied to a second bucket for the same reason. - - A key that is not in `allowed` therefore reaches the store and never reaches a screen — - which also means a stuffed key cannot be used to discover what exists: it comes back only - if the session could already see it. - """ - try: - stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} - except Exception: - return [] # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return [] - out = [] - for key, at in stored.items(): - key = str(key) - if allowed is not None and key not in allowed: - continue - try: - at = int(at) - except (TypeError, ValueError): - continue # a stamp this build cannot read is not a stamp - if at <= 0: - continue - out.append({"key": key, "at": at}) - out.sort(key=lambda r: r["at"], reverse=True) - return out[:_MAX_RECENTS] - - -# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── -# -# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the -# block that computed them (`_important_counts`) went with them. It read one store bucket PER -# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database -# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on -# **14** pages. It is **0 and 0** now. -# -# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the -# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold -# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. -# A budget that can be exhausted is a number that can be silently short, which is why it also had to -# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to -# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge -# instead of a late rail. Closes D-288 and D-289. -# -# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the -# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store -# read was — and duplicating them into the new route would have put two answers behind one badge. - -#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the -#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> -#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is -#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second -#: copy here would be free to drift from it ([[one-question-two-normalizers]]). -_WS_SUFFIX = "_table_workspace" - - -def _visible_views(doc, uname, is_admin, granted_ids): - """Every view on ONE database that THIS caller can see, from an already-read workspace doc. - - ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get - wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` - — one home per view, never both — so "how many views are marked important here" has a - DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and - `admin` has none, on the same database. A tenant-wide count would put a number in the owner's - rail that no view sidebar they can open would ever add up to. - - ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for - it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole - bucket per database (the N whole reads this block exists to avoid), and the only other option is - a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told - exists; a private import cannot. - """ - import core.table_store as table_store - out = {} - for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): - if isinstance(view, dict): - out[str(vid)] = view - for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): - if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): - out.setdefault(str(vid), view) - # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the - # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. - if granted_ids: - for stratum, blob in doc.items(): - if stratum == uname or not isinstance(blob, dict): - continue - for vid, view in (blob.get("views") or {}).items(): - if str(vid) in granted_ids and isinstance(view, dict): - out.setdefault(str(vid), view) - return out - - -def _view_record_count(cfg, cohorts): - """How many RECORDS this view resolves to, or None when that cannot be answered for free. - - ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact - because the view CARRIES its row set; the third — an ordinary filtered view — can only be - counted by running its filters over the records, and the records are the one thing this route - must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` - here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and - the database's `partial` flag says so, which is the whole of R6's second sentence applied to a - badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a - number that is short by an unknown amount. - - ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` - counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); - this end is the exact inverse — it has no rows at all and no window either. The two are honest - about different halves, which is why `partial` had to be on the wire rather than derived. - """ - if not isinstance(cfg, dict): - return None - # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` - # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. - lock = str(cfg.get("cohortLock") or "").strip() - if lock: - n = cohorts.get(lock) - return int(n) if isinstance(n, int) else None - # A curated row set carries its own count. - pids = cfg.get("memberPids") - if isinstance(pids, list) and pids: - return len(pids) - return None - - -def _visible_database_keys(session): - """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. - - ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: - - · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT - CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. - A star scan cannot borrow that: it would open the view bucket of a database this workspace's - catalogue does not include. `nav()` applies the catalogue filter; that helper never has. - · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so - it cannot be reached from another route without rebuilding the payload. - - So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same - order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the - tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes - the answer honest for a caller about to read a view bucket: a module surface with no workspace - (`sales`, `ar`) is not a database and has no views to star. - - ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` - excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: - a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even - though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious - reading — this ticket's first draft assumed the clause was excluding children and wrote a - docstring around a defect that does not exist. - - ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as - authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. - """ - import core.view_templates as view_templates - keys, enumerated = set(), True - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} - for p in (perms.nav_pages(session.user) or []): - key = str(p.get("key") or "") - if allowed is not None and key not in allowed \ - and str(p.get("parent") or "") not in allowed: - continue - if view_templates.workspace_key(key): - keys.add(key) - try: - import core.user_tables as user_tables - # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection - # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - k = str(e.get("key") or "") - if k: - keys.add(k) - except Exception: - enumerated = False - return keys, enumerated - - -def _granted_view_ids(session): - """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. - - ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above - takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with - the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to - answer "which views has this person starred", so the derivation moves here beside the reader - it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). - - ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming - me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to - agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test - would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so - it costs a dict lookup per id. - ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. - """ - import core.shares as shares - try: - return {str(v) for v in - ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) - .get("view") or []) - if shares.role_for("view", str(v), session.uname, - st=session.runtime) in ("view", "edit")} - except Exception: - return set() - - -@router.get("/nav") -def nav(session: Session = Depends(require_session)): - """`{pages: [{key,label,source?,chrome}], landing}`. - - Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you - are not allowed". A session that may open nothing at all is a misconfigured account, and it - gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is - how a permission bug hides in plain sight (X2's never-an-empty-200 rule). - """ - pages = list(perms.nav_pages(session.user) or []) - # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. - # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything - # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is - # its own databases, which is what "different set of databases at later waves" means. - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. - # - # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all - # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs - # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a - # row it removes and a row a store failure dropped are the SAME absence on the wire, and the - # client renders `null` for both — pixel-identical to still-loading. Naming the removals is - # what lets the shell tell "not part of this workspace" from "we could not read it". - _omitted = [] - if tmods != "all": - allowed = {str(k) for k in (tmods or [])} - _omitted = sorted({str(p.get("key")) for p in pages - if p.get("key") not in allowed - and (p.get("parent") or "") not in allowed}) - pages = [p for p in pages - if p.get("key") in allowed or (p.get("parent") or "") in allowed] - # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — - # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so - # the nav cannot offer a row the table routes would refuse. - # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. - # - # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one - # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below - # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms - # in-process** — the second figure is the one that matters, because with the network gone and - # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every - # other collapses to 20–161 ms. That is per-call CPU, and this is it. - # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ - # manage loop below consumes this same dict. - _ut_defs, _ut_locked_mode, _degraded = {}, "", [] - try: - import core.user_tables as user_tables - # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below - # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and - # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the - # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of - # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms - # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds - # apart because of CSS or ordering, they arrived when their route finished copying. - # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that - # needs a row, it raises with the reason instead of painting an empty grid. Use - # `all_tables` for that, and know you are buying the whole copy back. - _ut_defs = user_tables.all_defs(st=session.runtime) or {} - # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its - # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked - # the moment this import failed. A sentinel that can equal real data is not a sentinel. - _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) - _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) - # ⛔ AND THE EDITOR'S DENY REACHES THIS HALF TOO. `nav_entries` is `may_open`-filtered - # — creator, admin or a share — which knows nothing about the access toggle in - # Manage user. An administrator unticking a shared database left it on the rail. - # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. - import core.perm_scope as _perm_scope - # ⭐⭐ WAVE 37 · T07 (D-267, PRD amendment A8) — A RETIRED KEY CANNOT REACH THE NAV, WHATEVER - # THE TENANT DOCUMENT STILL CARRIES. - # - # W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` / - # `product_data` — ONE SUBJECT, ONE DATABASE — and `apply_plan` pops them from a tenant's - # document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live - # label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the - # PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the - # rebuild being scoped to `is_royal`). - # - # ⛔ SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN - # BLURRED: the stale ROW may still sit in a tenant's document, and removing it is - # `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document - # which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that - # migration has not reached them. A defect whose fix depends on a migration running is a - # defect that comes back on the one tenant the migration missed - # [[a-migration-that-runs-on-the-next-write]]. - # - # ⚠ `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift - # this guard exists to prevent, and it would go stale the first time a third key is retired. - # ⚠ Inside the same `try:` as the rest of the user-table read on purpose — if the import - # fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A - # guard that silently stops guarding is worse than one that says it could not run. - try: - from odoo_relational import RETIRED_KEYS as _retired - except Exception: - _retired = () - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=_lent): - if e["key"] in _retired: - continue - if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): - continue - pages.append({"key": e["key"], "label": e["label"], - "source": e.get("source") or "Blank", "chrome": "main"}) - except Exception: - # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. - # - # A store blip must not take the whole nav down with it, and that half stands. What - # changed is that it used to answer **200 OK with every database missing** and say - # nothing — so a client cannot tell "this tenant has no databases" from "we could not - # read them", and the rail renders the same complete-looking thing either way. That is - # the owner's *"Connectors and Automation module still disappears"* class of report: - # the payload is a claim about what exists, and a claim it could not verify has to be - # marked as such. `degraded` is that mark; the shell renders it in the affected slot. - _degraded.append("databases") - pass - if not pages: - if tmods != "all": - # A provisioned tenant with no modules and no databases YET is a legitimate empty - # state, not a misconfigured account — the client renders "create your first - # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is - # empty rather than leaving 200-[] ambiguous. - return {"pages": [], "landing": None, "empty": "no_databases"} - raise err(403, "no_surfaces", - "your account has no dashboards assigned. Ask an administrator.") - # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry - # rows, after the tenant module filter, after the user tables. Merged HERE rather than - # applied by the client for one reason: `label` is what every reader of this payload shows, - # including the Settings modal's `moduleLabels` map, so a client-side merge would put the - # override in one door and the registry label in the other. - # - # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A - # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If - # either ever starts handing back cached or module-level dicts, this loop would write one - # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant - # leak with no symptom until two tenants rename the same registry key. Copy the rows before - # merging on the day that invariant changes. - meta = _read_nav_meta(session.runtime) - # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. - # - # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS - # ORDER so a cold container would reach the databases somebody actually works in first. Every - # one of those was a correct answer to the wrong question: the block was `N` per-database store - # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block - # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per - # `/nav`** before this ticket, **0 and 0** after. - # - # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer - # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. - # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave - # 23), and it was only computed this early so the deleted budget could spend itself in its order. - _page_keys = {str(p.get("key", "")) for p in pages} - recents = _read_recents(session.runtime, session.uname, _page_keys) - # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. - # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. - # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, - # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. - # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an - # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. - for p in pages: - # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because - # the server is the only end that knows — the client cannot see who created a user - # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control - # only where the write would actually land, and the route re-checks it regardless. - # - # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the - # de5037f share-grant admission widened `may_open`, so a row's presence now includes - # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are - # therefore answered from the DEFINITION: creator-or-admin strictly, matching the - # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no - # controls — the honest shape (the old code offered rename to users the route 403'd). - key = str(p.get("key", "")) - if key.startswith("ut_"): - _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") - _mine = bool(session.admin or (_creator and _creator == session.uname)) - p["manage"] = _mine - p["canDelete"] = _mine - # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. - # - # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md - # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still - # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the - # live example; their rows arrive from the engine, so a "+" row there could only - # refuse, which R8 calls a fake affordance. - # - # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is - # inlined only because `_ut_defs` is already in hand — calling the predicate per row - # would be one store read per database on every nav request — and the VALUE comes - # from the module's own constant rather than a copied string, so the two cannot drift - # to different answers. If that predicate ever grows a second condition, this must - # become a call. - # - # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is - # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while - # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs - # a missing hint, never a write that should not have landed. - if (_ut_locked_mode - and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): - p["locked"] = True - elif session.admin: - p["manage"] = True - # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database - # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer - # never had to test for the key. R4 retires the count from the rail and the flyout entirely - # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. - # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an - # absent key reads as `undefined` there, which is the same thing the optional field already - # meant for a non-database row. Flagged to A rather than assumed harmless. - entry = meta.get(p.get("key")) if meta else None - if not entry: - continue - if entry.get("icon"): - p["icon"] = entry["icon"] - if entry.get("name"): - p["label"] = entry["name"] - landing = perms.landing_page(session.user) - if landing and not any(p.get("key") == landing for p in pages): - landing = pages[0].get("key") - # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks - # for. A second round trip for a list this short, computed from a bucket this route is - # already holding the store open for, would be a request per page load for no gain. - # - # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases - # included. Pruning against `perms.nav_pages` alone would silently drop every user database - # from Home's recents: the exact surface R7 is about, invisible, with every gate green. - # - # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — - # had the identical bug and nobody connected the two for four waves. It is fixed at the - # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so - # both doors finally agree on what a placeable key is. This block keeps using the assembled - # list because it already holds it: re-enumerating here would be a second store read for an - # answer sitting in a local variable. - # - # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, - # because the important-count block spends its budget in RECENTS ORDER (see there). It is still - # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here - # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. - # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. - # `omitted` — this workspace's catalogue does not include these modules. Deliberate. - # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says - # so in the affected slot instead of rendering a confident nothing. - # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as - # `[]` rather than being omitted when empty, which is the same rule `limits` follows. - return {"pages": pages, "landing": landing, "recents": recents, - "omitted": _omitted, "degraded": _degraded} - - -@router.post("/nav/opened") -def nav_opened(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. - - The client calls this on every route commit, so this is the only write in this file on a - hot path, and three things follow from that: - - · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per - page open against an HF-Dataset-backed store would put a network round trip inside every - navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; - this is. - · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string - would make the stamp cost more than the navigation that triggered it — and it would buy - nothing, because the READ prunes to what the session may currently see. An unknown or - revoked key is stored and never served back. - · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client - grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to - go, which is the same rule the read applies. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip()[:60] - if not key: - raise err(400, "bad_request", "no page was named") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was recorded.") - stamp, uname = _now(), session.uname - - def _up(data): - data = data if isinstance(data, dict) else {} - mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} - mine[key] = stamp - if len(mine) > _MAX_RECENTS: - # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape - # sorts as 0 and is the first thing evicted, which is the right answer for a value - # this route can no longer read. - def _at(item): - try: - return int(item[1]) - except (TypeError, ValueError): - return 0 - mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) - data[uname] = mine - return data - - try: - session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was recorded.") - return {"key": key, "at": stamp} - - -@router.post("/nav/meta") -def save_nav_meta(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. - - A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the - asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so - replacing the document whole is what makes a deleted folder stay deleted. This bucket is - shared by every admin in the tenant: a wholesale write here means whoever saves last - silently erases what the other one named while their tab was open. - - THREE WALLS, all fail-closed: - · the SESSION must be able to open the key at all (the same predicate the nav uses, so - the rail and this route cannot disagree about what exists); - · the WRITE is admin-only — renaming a database is a change every user in the tenant - sees, which is the definition of an administrative act here; - · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry - literals: honouring an override would leave this payload and `core/registry.py` - calling the same module two different things, and the wave-16 lesson is that the - client must not be the place that decides what a payload meant. - - `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an - icon change two independent writes rather than a race between them. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip() - if not key: - raise err(400, "bad_request", "no database was named") - # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── - # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. - # - # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what - # `user_tables.may_open` already means. A database you made is yours to name; requiring - # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` - # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, - # because a user table is not a module. Same split `/nav/schema/{key}` makes. - # · everything else — admin only, and icon only. There is no owner of `customer_data` to - # defer to, and its label is a compiled registry literal (refused below regardless). +"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. + +SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who +may see what — a nav that hides a link the API would still serve is a UI courtesy, not a +permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), +so the nav and the 403 can never disagree about a grant. + +The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no +`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the +`prefs.json` Library preference is deliberately NOT applied — is documented on +`core.perms.nav_pages`, which is where it belongs: one place, both callers. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, perms, require_session + +router = APIRouter(prefix="/api/v1") + +#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder +#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a +#: surface (the server-filtered nav still decides what exists). +_NAV_PREFS_KEY = "nav_prefs" +_MAX_NAV_FOLDERS = 16 + +#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. +#: +#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a +#: separate bucket rather than another field in that one. Folders and placement are one +#: person's arrangement of their own rail — per-user by definition. What a database is CALLED +#: and what it looks like are facts about the database: a workspace where two people call the +#: same table different things has no shared vocabulary left to discuss it in. Same store, two +#: buckets, because they answer to two different owners. +#: +#: Shape: {"": {"icon": {"shape": , "tone": }, +#: "name": ""}} +_NAV_META_KEY = "nav_meta" + +#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these +#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, +#: so it is the one place the list is written twice. +#: +#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by +#: shape and maps the result, so a shape this whitelist let through and the renderer does not +#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the +#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of +#: that. If the two lists ever drift the symptom is an icon that silently reverts to the +#: default, which is the loudest SAFE failure available here. +_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", + "chart", "map", "users", "clock", "heart", "bolt"}) +_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) +_MAX_NAV_NAME = 60 + +#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. +#: +#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is +#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature +#: rather than a convenience one. +#: +#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. +#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An +#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one +#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls +#: off the list while forty slots hold repeat visits to the first. Keying by page makes +#: "recent" mean what the word means, and makes the cap bound the number of DATABASES +#: remembered rather than the number of clicks. +_NAV_RECENTS_KEY = "nav_recents" +_MAX_RECENTS = 50 + +#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a +#: precedent, not a preference. `user_tables.create` writes +#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses +#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 +#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — +#: with nothing to go red, because both ends are internally consistent. An integer instant has +#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect +#: shipped; this is that lesson applied before it.) +def _now() -> int: + return int(time.time()) + + +def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): + """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. + + `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown + or invisible key is dropped (it can return when the grant does — placement is cosmetic, + so pruning is loss-free). Unknown folder refs drop the placement, not the folder. + + `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the + `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than + pruned: keeping a placement whose database may since have been deleted is cosmetically + harmless, while pruning a live one is the silent data loss this function just stopped + causing. Never widened to non-`ut_` keys — those come from the compiled registry, which + cannot fail to enumerate. + """ + raw = raw if isinstance(raw, dict) else {} + folders, seen = [], set() + for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:40] + name = " ".join(str(f.get("name") or "").split())[:40] + if not fid or fid in seen or not name: + continue + seen.add(fid) + folders.append({"id": fid, "name": name}) + ids = {f["id"] for f in folders} + placement = {} + src = raw.get("placement") + if isinstance(src, dict): + for k, v in src.items(): + k, v = str(k)[:60], str(v)[:40] + if v not in ids: + continue # the folder is gone; the placement goes with it + if page_keys is None or k in page_keys: + placement[k] = v + elif keep_unknown_ut and k.startswith("ut_"): + placement[k] = v + return {"folders": folders, "placement": placement} + + +def _placeable_top_keys(session): + """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. + + ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by + construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and + `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's + user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. + So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement + naming one — on READ and on WRITE. + + The symptom was "a database I drag into a folder falls out of it later", never "it does not + work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy + (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. + The next reload served the pruned copy and the database was back at root. Built-in modules + (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly + why it read as intermittent rather than as a missing feature. + + ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents + block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the + ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 + fix did not reach. + + `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set + as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement + because the store blinked would be the original defect wearing a different cause. + """ + pages = perms.nav_pages(session.user) or [] + keys = {p["key"] for p in pages if not p.get("parent")} + try: + import core.user_tables as user_tables + # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never + # name a database this session cannot see, and the nav and this door agree by sharing + # one predicate rather than by two lists being kept in step. + # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is + # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires + # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured + # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the + # owner reports as "the Automation row arrives ten seconds late". + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + keys.add(str(e.get("key") or "")) + except Exception: + return keys, False + return keys, True + + +def _clean_icon(raw): + """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" + if not isinstance(raw, dict): + return None + shape, tone = raw.get("shape"), raw.get("tone") + if shape not in _ICON_SHAPES or tone not in _ICON_TONES: + return None + return {"shape": shape, "tone": tone} + + +def _read_nav_meta(runtime): + """The tenant's `nav_meta`, validated on the way OUT as well as in. + + Re-validating a read looks redundant and is not: the bucket outlives this code, an older + build may have written a shape this one no longer accepts, and the whitelist is mirrored + from a vocabulary that lives in another language. A row whose icon does not survive + validation renders the default mark — never a crash, and never a half-drawn glyph. + """ + try: + stored = runtime.get(_NAV_META_KEY) or {} + except Exception: + return {} # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return {} + out = {} + for key, meta in stored.items(): + if not isinstance(meta, dict): + continue + entry = {} + icon = _clean_icon(meta.get("icon")) + if icon: + entry["icon"] = icon + # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write + # route refuses a name for a built-in key, but a record written by an older build (or + # by hand) could still carry one — and honouring it would let the store rename + # `customer_data` on screen while `core/registry.py` and every other reader went on + # calling it Customer. A name that could not be written today is not served today. + name = meta.get("name") + if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): + entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] + if entry: + out[str(key)] = entry + return out + + +def _read_recents(runtime, uname, allowed): + """This user's recents, newest first, PRUNED to what they may currently see. + + ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: + + · the WRITE happens on every route open — it is the one hot path this file has — so it + must not build the whole nav to validate one key; + · a key whose grant was REVOKED must stop being offered without anyone running a + migration, and must come back if the grant does. That is `_clean_nav_prefs`' own + prune-never-invent posture, applied to a second bucket for the same reason. + + A key that is not in `allowed` therefore reaches the store and never reaches a screen — + which also means a stuffed key cannot be used to discover what exists: it comes back only + if the session could already see it. + """ + try: + stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} + except Exception: + return [] # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return [] + out = [] + for key, at in stored.items(): + key = str(key) + if allowed is not None and key not in allowed: + continue + try: + at = int(at) + except (TypeError, ValueError): + continue # a stamp this build cannot read is not a stamp + if at <= 0: + continue + out.append({"key": key, "at": at}) + out.sort(key=lambda r: r["at"], reverse=True) + return out[:_MAX_RECENTS] + + +# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── +# +# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the +# block that computed them (`_important_counts`) went with them. It read one store bucket PER +# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database +# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on +# **14** pages. It is **0 and 0** now. +# +# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the +# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold +# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. +# A budget that can be exhausted is a number that can be silently short, which is why it also had to +# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to +# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge +# instead of a late rail. Closes D-288 and D-289. +# +# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the +# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store +# read was — and duplicating them into the new route would have put two answers behind one badge. + +#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the +#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> +#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is +#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second +#: copy here would be free to drift from it ([[one-question-two-normalizers]]). +_WS_SUFFIX = "_table_workspace" + + +def _visible_views(doc, uname, is_admin, granted_ids): + """Every view on ONE database that THIS caller can see, from an already-read workspace doc. + + ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get + wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` + — one home per view, never both — so "how many views are marked important here" has a + DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and + `admin` has none, on the same database. A tenant-wide count would put a number in the owner's + rail that no view sidebar they can open would ever add up to. + + ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for + it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole + bucket per database (the N whole reads this block exists to avoid), and the only other option is + a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told + exists; a private import cannot. + """ + import core.table_store as table_store + out = {} + for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): + if isinstance(view, dict): + out[str(vid)] = view + for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): + if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): + out.setdefault(str(vid), view) + # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the + # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. + if granted_ids: + for stratum, blob in doc.items(): + if stratum == uname or not isinstance(blob, dict): + continue + for vid, view in (blob.get("views") or {}).items(): + if str(vid) in granted_ids and isinstance(view, dict): + out.setdefault(str(vid), view) + return out + + +def _view_record_count(cfg, cohorts): + """How many RECORDS this view resolves to, or None when that cannot be answered for free. + + ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact + because the view CARRIES its row set; the third — an ordinary filtered view — can only be + counted by running its filters over the records, and the records are the one thing this route + must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` + here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and + the database's `partial` flag says so, which is the whole of R6's second sentence applied to a + badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a + number that is short by an unknown amount. + + ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` + counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); + this end is the exact inverse — it has no rows at all and no window either. The two are honest + about different halves, which is why `partial` had to be on the wire rather than derived. + """ + if not isinstance(cfg, dict): + return None + # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` + # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + n = cohorts.get(lock) + return int(n) if isinstance(n, int) else None + # A curated row set carries its own count. + pids = cfg.get("memberPids") + if isinstance(pids, list) and pids: + return len(pids) + return None + + +def _visible_database_keys(session): + """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. + + ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: + + · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT + CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. + A star scan cannot borrow that: it would open the view bucket of a database this workspace's + catalogue does not include. `nav()` applies the catalogue filter; that helper never has. + · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so + it cannot be reached from another route without rebuilding the payload. + + So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same + order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the + tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes + the answer honest for a caller about to read a view bucket: a module surface with no workspace + (`sales`, `ar`) is not a database and has no views to star. + + ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` + excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: + a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even + though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious + reading — this ticket's first draft assumed the clause was excluding children and wrote a + docstring around a defect that does not exist. + + ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as + authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. + """ + import core.view_templates as view_templates + keys, enumerated = set(), True + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} + for p in (perms.nav_pages(session.user) or []): + key = str(p.get("key") or "") + if allowed is not None and key not in allowed \ + and str(p.get("parent") or "") not in allowed: + continue + if view_templates.workspace_key(key): + keys.add(key) + try: + import core.user_tables as user_tables + # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection + # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + k = str(e.get("key") or "") + if k: + keys.add(k) + except Exception: + enumerated = False + return keys, enumerated + + +def _granted_view_ids(session): + """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. + + ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above + takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with + the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to + answer "which views has this person starred", so the derivation moves here beside the reader + it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). + + ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming + me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to + agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test + would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so + it costs a dict lookup per id. + ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. + """ + import core.shares as shares + try: + return {str(v) for v in + ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) + .get("view") or []) + if shares.role_for("view", str(v), session.uname, + st=session.runtime) in ("view", "edit")} + except Exception: + return set() + + +@router.get("/nav") +def nav(session: Session = Depends(require_session)): + """`{pages: [{key,label,source?,chrome}], landing}`. + + Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you + are not allowed". A session that may open nothing at all is a misconfigured account, and it + gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is + how a permission bug hides in plain sight (X2's never-an-empty-200 rule). + """ + pages = list(perms.nav_pages(session.user) or []) + # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. + # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything + # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is + # its own databases, which is what "different set of databases at later waves" means. + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. + # + # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all + # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs + # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a + # row it removes and a row a store failure dropped are the SAME absence on the wire, and the + # client renders `null` for both — pixel-identical to still-loading. Naming the removals is + # what lets the shell tell "not part of this workspace" from "we could not read it". + _omitted = [] + if tmods != "all": + allowed = {str(k) for k in (tmods or [])} + _omitted = sorted({str(p.get("key")) for p in pages + if p.get("key") not in allowed + and (p.get("parent") or "") not in allowed}) + pages = [p for p in pages + if p.get("key") in allowed or (p.get("parent") or "") in allowed] + # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — + # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so + # the nav cannot offer a row the table routes would refuse. + # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. + # + # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one + # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below + # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms + # in-process** — the second figure is the one that matters, because with the network gone and + # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every + # other collapses to 20–161 ms. That is per-call CPU, and this is it. + # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ + # manage loop below consumes this same dict. + _ut_defs, _ut_locked_mode, _degraded = {}, "", [] + try: + import core.user_tables as user_tables + # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below + # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and + # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the + # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of + # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms + # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds + # apart because of CSS or ordering, they arrived when their route finished copying. + # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that + # needs a row, it raises with the reason instead of painting an empty grid. Use + # `all_tables` for that, and know you are buying the whole copy back. + _ut_defs = user_tables.all_defs(st=session.runtime) or {} + # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its + # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked + # the moment this import failed. A sentinel that can equal real data is not a sentinel. + _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) + _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) + # ⛔ AND THE EDITOR'S DENY REACHES THIS HALF TOO. `nav_entries` is `may_open`-filtered + # — creator, admin or a share — which knows nothing about the access toggle in + # Manage user. An administrator unticking a shared database left it on the rail. + # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. + import core.perm_scope as _perm_scope + # ⭐⭐ WAVE 37 · T07 (D-267, PRD amendment A8) — A RETIRED KEY CANNOT REACH THE NAV, WHATEVER + # THE TENANT DOCUMENT STILL CARRIES. + # + # W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` / + # `product_data` — ONE SUBJECT, ONE DATABASE — and `apply_plan` pops them from a tenant's + # document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live + # label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the + # PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the + # rebuild being scoped to `is_royal`). + # + # ⛔ SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN + # BLURRED: the stale ROW may still sit in a tenant's document, and removing it is + # `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document + # which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that + # migration has not reached them. A defect whose fix depends on a migration running is a + # defect that comes back on the one tenant the migration missed + # [[a-migration-that-runs-on-the-next-write]]. + # + # ⚠ `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift + # this guard exists to prevent, and it would go stale the first time a third key is retired. + # ⚠ Inside the same `try:` as the rest of the user-table read on purpose — if the import + # fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A + # guard that silently stops guarding is worse than one that says it could not run. + try: + from odoo_relational import RETIRED_KEYS as _retired + except Exception: + _retired = () + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=_lent): + if e["key"] in _retired: + continue + if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): + continue + pages.append({"key": e["key"], "label": e["label"], + "source": e.get("source") or "Blank", "chrome": "main"}) + except Exception: + # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. + # + # A store blip must not take the whole nav down with it, and that half stands. What + # changed is that it used to answer **200 OK with every database missing** and say + # nothing — so a client cannot tell "this tenant has no databases" from "we could not + # read them", and the rail renders the same complete-looking thing either way. That is + # the owner's *"Connectors and Automation module still disappears"* class of report: + # the payload is a claim about what exists, and a claim it could not verify has to be + # marked as such. `degraded` is that mark; the shell renders it in the affected slot. + _degraded.append("databases") + pass + if not pages: + if tmods != "all": + # A provisioned tenant with no modules and no databases YET is a legitimate empty + # state, not a misconfigured account — the client renders "create your first + # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is + # empty rather than leaving 200-[] ambiguous. + return {"pages": [], "landing": None, "empty": "no_databases"} + raise err(403, "no_surfaces", + "your account has no dashboards assigned. Ask an administrator.") + # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry + # rows, after the tenant module filter, after the user tables. Merged HERE rather than + # applied by the client for one reason: `label` is what every reader of this payload shows, + # including the Settings modal's `moduleLabels` map, so a client-side merge would put the + # override in one door and the registry label in the other. + # + # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A + # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If + # either ever starts handing back cached or module-level dicts, this loop would write one + # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant + # leak with no symptom until two tenants rename the same registry key. Copy the rows before + # merging on the day that invariant changes. + meta = _read_nav_meta(session.runtime) + # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. + # + # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS + # ORDER so a cold container would reach the databases somebody actually works in first. Every + # one of those was a correct answer to the wrong question: the block was `N` per-database store + # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block + # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per + # `/nav`** before this ticket, **0 and 0** after. + # + # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer + # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. + # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave + # 23), and it was only computed this early so the deleted budget could spend itself in its order. + _page_keys = {str(p.get("key", "")) for p in pages} + recents = _read_recents(session.runtime, session.uname, _page_keys) + # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. + # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. + # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, + # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. + # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an + # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. + for p in pages: + # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because + # the server is the only end that knows — the client cannot see who created a user + # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control + # only where the write would actually land, and the route re-checks it regardless. + # + # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the + # de5037f share-grant admission widened `may_open`, so a row's presence now includes + # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are + # therefore answered from the DEFINITION: creator-or-admin strictly, matching the + # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no + # controls — the honest shape (the old code offered rename to users the route 403'd). + key = str(p.get("key", "")) + if key.startswith("ut_"): + _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") + _mine = bool(session.admin or (_creator and _creator == session.uname)) + p["manage"] = _mine + p["canDelete"] = _mine + # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. + # + # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md + # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still + # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the + # live example; their rows arrive from the engine, so a "+" row there could only + # refuse, which R8 calls a fake affordance. + # + # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is + # inlined only because `_ut_defs` is already in hand — calling the predicate per row + # would be one store read per database on every nav request — and the VALUE comes + # from the module's own constant rather than a copied string, so the two cannot drift + # to different answers. If that predicate ever grows a second condition, this must + # become a call. + # + # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is + # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while + # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs + # a missing hint, never a write that should not have landed. + if (_ut_locked_mode + and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): + p["locked"] = True + elif session.admin: + p["manage"] = True + # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database + # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer + # never had to test for the key. R4 retires the count from the rail and the flyout entirely + # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. + # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an + # absent key reads as `undefined` there, which is the same thing the optional field already + # meant for a non-database row. Flagged to A rather than assumed harmless. + entry = meta.get(p.get("key")) if meta else None + if not entry: + continue + if entry.get("icon"): + p["icon"] = entry["icon"] + if entry.get("name"): + p["label"] = entry["name"] + landing = perms.landing_page(session.user) + if landing and not any(p.get("key") == landing for p in pages): + landing = pages[0].get("key") + # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks + # for. A second round trip for a list this short, computed from a bucket this route is + # already holding the store open for, would be a request per page load for no gain. + # + # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases + # included. Pruning against `perms.nav_pages` alone would silently drop every user database + # from Home's recents: the exact surface R7 is about, invisible, with every gate green. + # + # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — + # had the identical bug and nobody connected the two for four waves. It is fixed at the + # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so + # both doors finally agree on what a placeable key is. This block keeps using the assembled + # list because it already holds it: re-enumerating here would be a second store read for an + # answer sitting in a local variable. + # + # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, + # because the important-count block spends its budget in RECENTS ORDER (see there). It is still + # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here + # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. + # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. + # `omitted` — this workspace's catalogue does not include these modules. Deliberate. + # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says + # so in the affected slot instead of rendering a confident nothing. + # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as + # `[]` rather than being omitted when empty, which is the same rule `limits` follows. + return {"pages": pages, "landing": landing, "recents": recents, + "omitted": _omitted, "degraded": _degraded} + + +@router.post("/nav/opened") +def nav_opened(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. + + The client calls this on every route commit, so this is the only write in this file on a + hot path, and three things follow from that: + + · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per + page open against an HF-Dataset-backed store would put a network round trip inside every + navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; + this is. + · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string + would make the stamp cost more than the navigation that triggered it — and it would buy + nothing, because the READ prunes to what the session may currently see. An unknown or + revoked key is stored and never served back. + · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client + grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to + go, which is the same rule the read applies. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip()[:60] + if not key: + raise err(400, "bad_request", "no page was named") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was recorded.") + stamp, uname = _now(), session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + mine[key] = stamp + if len(mine) > _MAX_RECENTS: + # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape + # sorts as 0 and is the first thing evicted, which is the right answer for a value + # this route can no longer read. + def _at(item): + try: + return int(item[1]) + except (TypeError, ValueError): + return 0 + mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) + data[uname] = mine + return data + + try: + session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was recorded.") + return {"key": key, "at": stamp} + + +@router.post("/nav/meta") +def save_nav_meta(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. + + A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the + asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so + replacing the document whole is what makes a deleted folder stay deleted. This bucket is + shared by every admin in the tenant: a wholesale write here means whoever saves last + silently erases what the other one named while their tab was open. + + THREE WALLS, all fail-closed: + · the SESSION must be able to open the key at all (the same predicate the nav uses, so + the rail and this route cannot disagree about what exists); + · the WRITE is admin-only — renaming a database is a change every user in the tenant + sees, which is the definition of an administrative act here; + · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry + literals: honouring an override would leave this payload and `core/registry.py` + calling the same module two different things, and the wave-16 lesson is that the + client must not be the place that decides what a payload meant. + + `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an + icon change two independent writes rather than a race between them. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip() + if not key: + raise err(400, "bad_request", "no database was named") + # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── + # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. + # + # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what + # `user_tables.may_open` already means. A database you made is yours to name; requiring + # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` + # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, + # because a user table is not a module. Same split `/nav/schema/{key}` makes. + # · everything else — admin only, and icon only. There is no owner of `customer_data` to + # defer to, and its label is a compiled registry literal (refused below regardless). if key.startswith("ut_"): import core.user_tables as user_tables # This write edits metadata, never a record. The table wall therefore needs its @@ -772,124 +772,124 @@ def save_nav_meta(body: dict = Body(default=None), definitions = user_tables.lend_defs(session.runtime) if not user_tables.get(key, st=definitions) or not user_tables.may_open( key, session.uname, session.admin, st=definitions): - raise err(403, "forbidden", "that database belongs to another user") - else: - if not session.admin: - raise err(403, "forbidden", - "only an administrator can change a built-in database's icon") - session.require(key) - - patch = {} - if "icon" in body: - icon = _clean_icon(body.get("icon")) - if body.get("icon") is not None and icon is None: - # Loud, not silent. A shape this build does not know is a CLIENT that has drifted - # from this whitelist, and answering 200 to a write that stored nothing is how - # that drift stays invisible until a user reports "my icon keeps resetting". - raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") - patch["icon"] = icon - if "name" in body: - if not key.startswith("ut_"): - raise err(400, "name_not_allowed", - "only a database you created can be renamed. This one's name comes " - "from the module registry") - name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] - if not name: - raise err(400, "bad_request", "a database needs a name") - patch["name"] = name - if not patch: - raise err(400, "bad_request", "nothing to change") - - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} - for field, value in patch.items(): - if value is None: - entry.pop(field, None) # an explicit null CLEARS - else: - entry[field] = value - # An entry with nothing left in it is removed rather than stored empty: the read side - # skips empties anyway, and a bucket that accumulates `{}` per key is a document that - # grows forever and says nothing. - if entry: - data[key] = entry - else: - data.pop(key, None) - return data - - try: - session.runtime.update(_NAV_META_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the change was not saved: the store refused the write.") - return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} - - -@router.get("/nav/prefs") -def nav_prefs(session: Session = Depends(require_session)): - """This user's database-list folders, re-validated at serve time against what they may - currently see (a revoked page's placement vanishes with the page, and returns with it).""" - try: - stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} - except Exception: - stored = {} - keys, enumerated = _placeable_top_keys(session) - return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} - - -@router.post("/nav/prefs") -def save_nav_prefs(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" - keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: - data[session.uname] = clean - else: - data.pop(session.uname, None) - return data - - try: - session.runtime.update(_NAV_PREFS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") - return {"prefs": clean} - - -@router.get("/nav/schema/{key}") -def nav_schema(key: str, session: Session = Depends(require_session)): - """The database's schema drawer payload: its field contract + the semantic measures this - session may build with. - - TWO WALLS, ANSWERING TWO QUESTIONS, AND THEY FAIL DIFFERENTLY ON PURPOSE. The DATABASE wall - is the same predicate the nav uses — a key this session may not open answers 403, never a - redacted schema. The FIELD wall runs *after* it, on a database this session may open, and - its whole job is to narrow: the drawer serves the columns this reader receives everywhere - else, and no more. - - ⭐⭐ W38-T18 — THE FIELD WALL IS NEW HERE AND THIS DOOR IS WHERE IT WAS MISSING. Every grid - door has stripped hidden columns from both wires since W36-T21; the schema drawer beside them - served the raw contract to anyone `require`/`may_open` admitted. Owner instruction 10 — - *"Metrics field lookback and value must abide by the Filter permissioning"* — is satisfied on - the picker side already (the condition kit takes `fields` as a PROP off a payload that is - walled upstream), so this route is the one place the same reader could still learn the name, - the label and the VALUE VOCABULARY of a column their grid refuses them. - """ - # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate - # (`may_open`) rather than the module grant machinery — `session.require` would 403 every - # ut key because a user table is deliberately not a module. + raise err(403, "forbidden", "that database belongs to another user") + else: + if not session.admin: + raise err(403, "forbidden", + "only an administrator can change a built-in database's icon") + session.require(key) + + patch = {} + if "icon" in body: + icon = _clean_icon(body.get("icon")) + if body.get("icon") is not None and icon is None: + # Loud, not silent. A shape this build does not know is a CLIENT that has drifted + # from this whitelist, and answering 200 to a write that stored nothing is how + # that drift stays invisible until a user reports "my icon keeps resetting". + raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") + patch["icon"] = icon + if "name" in body: + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database you created can be renamed. This one's name comes " + "from the module registry") + name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] + if not name: + raise err(400, "bad_request", "a database needs a name") + patch["name"] = name + if not patch: + raise err(400, "bad_request", "nothing to change") + + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + for field, value in patch.items(): + if value is None: + entry.pop(field, None) # an explicit null CLEARS + else: + entry[field] = value + # An entry with nothing left in it is removed rather than stored empty: the read side + # skips empties anyway, and a bucket that accumulates `{}` per key is a document that + # grows forever and says nothing. + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + session.runtime.update(_NAV_META_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} + + +@router.get("/nav/prefs") +def nav_prefs(session: Session = Depends(require_session)): + """This user's database-list folders, re-validated at serve time against what they may + currently see (a revoked page's placement vanishes with the page, and returns with it).""" + try: + stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} + except Exception: + stored = {} + keys, enumerated = _placeable_top_keys(session) + return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} + + +@router.post("/nav/prefs") +def save_nav_prefs(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Wholesale replace, like the table folder stratum — the client sent its complete + picture, validated here; a partial merge would resurrect deleted folders forever.""" + keys, enumerated = _placeable_top_keys(session) + clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + if clean["folders"] or clean["placement"]: + data[session.uname] = clean + else: + data.pop(session.uname, None) + return data + + try: + session.runtime.update(_NAV_PREFS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the folder change was not saved: the store refused the write.") + return {"prefs": clean} + + +@router.get("/nav/schema/{key}") +def nav_schema(key: str, session: Session = Depends(require_session)): + """The database's schema drawer payload: its field contract + the semantic measures this + session may build with. + + TWO WALLS, ANSWERING TWO QUESTIONS, AND THEY FAIL DIFFERENTLY ON PURPOSE. The DATABASE wall + is the same predicate the nav uses — a key this session may not open answers 403, never a + redacted schema. The FIELD wall runs *after* it, on a database this session may open, and + its whole job is to narrow: the drawer serves the columns this reader receives everywhere + else, and no more. + + ⭐⭐ W38-T18 — THE FIELD WALL IS NEW HERE AND THIS DOOR IS WHERE IT WAS MISSING. Every grid + door has stripped hidden columns from both wires since W36-T21; the schema drawer beside them + served the raw contract to anyone `require`/`may_open` admitted. Owner instruction 10 — + *"Metrics field lookback and value must abide by the Filter permissioning"* — is satisfied on + the picker side already (the condition kit takes `fields` as a PROP off a payload that is + walled upstream), so this route is the one place the same reader could still learn the name, + the label and the VALUE VOCABULARY of a column their grid refuses them. + """ + # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate + # (`may_open`) rather than the module grant machinery — `session.require` would 403 every + # ut key because a user table is deliberately not a module. if key.startswith("ut_"): import core.user_tables as user_tables # Schema is definition-only by contract. Do not pay a whole `user_tables` read merely @@ -898,99 +898,99 @@ def nav_schema(key: str, session: Session = Depends(require_session)): defn = user_tables.get(key, st=definitions) if not defn or not user_tables.may_open(key, session.uname, session.admin, st=definitions): - raise err(403, "forbidden", "that database belongs to another user") - # ⭐⭐ W38-T18 — THE FIELD WALL ON A `ut_*` DATABASE, WHICH THIS DOOR HAS NEVER RUN. - # `may_open` answers *IF* you reach the database and says nothing about WHICH COLUMNS, so - # this drawer handed `defn["fields"]` whole to every account it admitted. The bypass is - # older than this ticket (it dates to the wall itself); what changed is the CATEGORY it - # leaks — since W38-T16 a hidden `ut_*` column means an administrator's `hiddenFields` - # *or* a column granted away, and a share door whose schema drawer still names the column - # tells the ungranted reader exactly what they were refused. - # ⚠ MERGED BEFORE THE WALL, NEVER AFTER — the same order `routes_tables.ut_assembly` - # runs, and for the reason `_ut_shared_fields` gives one door over: the closure must see - # the WHOLE contract, or a definition field whose formula reads a hidden tenant-wide - # column sits outside its reach and carries that value out wearing a second name. - # ⛔ ONLY THE DEFINITION'S OWN FIELDS ARE RETURNED. This drawer has never published the - # shared stratum and this ticket is not where it starts; widening the list handed to the - # closure can only ever ADD to the hidden set, never remove from it. - import routes_tables as _rt - base = list(defn.get("fields") or []) - hidden = _rt._ut_hidden(session, key, - _rt._ut_shared_fields(session, key, base), - st=session.runtime) - # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a - # schema panel opened from it that says another is the drift a rename is supposed to - # remove, not create. - return {"key": key, - "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") - or defn.get("label") or key), - "source": defn.get("source") or "Blank", - "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], - "source": f.get("source") or "overlay", - "description": str(f.get("description") or "")} - for f in base if f.get("key") not in hidden], - "measures": []} - session.require(key) - pages = perms.nav_pages(session.user) or [] - page = next((p for p in pages if p.get("key") == key), None) - if page is None: - raise err(404, "unknown_page", f"{key!r} is not a database this session can see") - fields = [] - if key in ("customer_data", "cohort", "customers"): - try: - import aios_grid - import routes_customers as _rc - # ⭐⭐ W38-T18 — THE FIELD WALL ON THE CUSTOMER CONTRACT, AND THE `options` LINE FOUR - # ROWS DOWN IS WHY IT IS A LEAK RATHER THAN AN UNTIDINESS. A schema drawer that names - # a hidden column has told the reader it exists; one that ships `options` has handed - # them the VALUE VOCABULARY of every choice and status column their grid refuses — - # the set of statuses the business uses, read straight off a door with no wall. - # ⛔ `_hidden_for` IS REUSED, NOT RE-DERIVED, and the difference is not cosmetic. - # `hidden_keys(user, MODULE, aios_grid.FIELDS)` is the obvious spelling and it is - # INCOMPLETE: that function's own docstring records that a wall computed from the - # canonical list alone answers "not hidden" for every runtime column, so the - # tenant-wide stratum never reaches the closure and a formula over a hidden column - # comes back. One evaluator, one caller, no second idea of what this session hides. - # ⚠ INSIDE THE `try`, DELIBERATELY. If the wall cannot be computed the route already - # answers with an empty contract rather than a full one — fail-closed is the only - # direction a schema door may fail, and the gate's ADMIN control is what stops that - # silent empty being mistaken for a wall that worked. - hidden = _rc._hidden_for(session) - for f in aios_grid.FIELDS: - if f["key"] in hidden: - continue - entry = {"key": f["key"], "label": f["label"], "type": f["type"], - "source": f["source"], - "description": str(f.get("description") or "")} - if f.get("options"): - entry["options"] = list(f["options"]) - fields.append(entry) - except Exception: - fields = [] - measures = [] - try: - from core import measure_resolve - import core.perm_scope as _perm_scope - team_id = perms.scope_team_id(session.user) - # ⭐⭐ W38-T19 — THE METRICS CAPABILITY REACHES THIS DRAWER TOO, AND IT IS THE ONE DOOR - # THE THREE GRID ASSEMBLIES DO NOT SPEAK FOR. This route calls `measure_resolve.offer` - # ITSELF rather than reading a `measures` key off an assembly, so a wall applied in - # `routes_customers` / `routes_products` / `routes_tables` narrows every grid and leaves - # the panel beside them naming the full lookback catalogue — the same one-door-short - # shape W38-T18 fixed for FIELDS in this very function, one key over. - # ⚠ THE UT_ BRANCH ABOVE RETURNS BEFORE THIS AND ALREADY SERVES `[]`, so there is - # nothing to narrow there; if it ever starts publishing an offer, it needs this line. - _offer = ((measure_resolve.offer(team_id) or []) - if _perm_scope.may_metrics(session.user, key) else []) - for m in _offer: - measures.append({"key": str(m.get("key") or ""), - "label": str(m.get("label") or m.get("key") or ""), - "type": str(m.get("type") or "")}) - except Exception: - measures = [] - out = {"key": key, "label": page.get("label") or key, - "source": page.get("source") or "", "fields": fields, "measures": measures} - if not fields: - # Honest, never a mock: a database whose contract is not yet published says so. - out["note"] = "This database has not published a field contract yet." - return out + raise err(403, "forbidden", "that database belongs to another user") + # ⭐⭐ W38-T18 — THE FIELD WALL ON A `ut_*` DATABASE, WHICH THIS DOOR HAS NEVER RUN. + # `may_open` answers *IF* you reach the database and says nothing about WHICH COLUMNS, so + # this drawer handed `defn["fields"]` whole to every account it admitted. The bypass is + # older than this ticket (it dates to the wall itself); what changed is the CATEGORY it + # leaks — since W38-T16 a hidden `ut_*` column means an administrator's `hiddenFields` + # *or* a column granted away, and a share door whose schema drawer still names the column + # tells the ungranted reader exactly what they were refused. + # ⚠ MERGED BEFORE THE WALL, NEVER AFTER — the same order `routes_tables.ut_assembly` + # runs, and for the reason `_ut_shared_fields` gives one door over: the closure must see + # the WHOLE contract, or a definition field whose formula reads a hidden tenant-wide + # column sits outside its reach and carries that value out wearing a second name. + # ⛔ ONLY THE DEFINITION'S OWN FIELDS ARE RETURNED. This drawer has never published the + # shared stratum and this ticket is not where it starts; widening the list handed to the + # closure can only ever ADD to the hidden set, never remove from it. + import routes_tables as _rt + base = list(defn.get("fields") or []) + hidden = _rt._ut_hidden(session, key, + _rt._ut_shared_fields(session, key, base), + st=session.runtime) + # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a + # schema panel opened from it that says another is the drift a rename is supposed to + # remove, not create. + return {"key": key, + "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") + or defn.get("label") or key), + "source": defn.get("source") or "Blank", + "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], + "source": f.get("source") or "overlay", + "description": str(f.get("description") or "")} + for f in base if f.get("key") not in hidden], + "measures": []} + session.require(key) + pages = perms.nav_pages(session.user) or [] + page = next((p for p in pages if p.get("key") == key), None) + if page is None: + raise err(404, "unknown_page", f"{key!r} is not a database this session can see") + fields = [] + if key in ("customer_data", "cohort", "customers"): + try: + import aios_grid + import routes_customers as _rc + # ⭐⭐ W38-T18 — THE FIELD WALL ON THE CUSTOMER CONTRACT, AND THE `options` LINE FOUR + # ROWS DOWN IS WHY IT IS A LEAK RATHER THAN AN UNTIDINESS. A schema drawer that names + # a hidden column has told the reader it exists; one that ships `options` has handed + # them the VALUE VOCABULARY of every choice and status column their grid refuses — + # the set of statuses the business uses, read straight off a door with no wall. + # ⛔ `_hidden_for` IS REUSED, NOT RE-DERIVED, and the difference is not cosmetic. + # `hidden_keys(user, MODULE, aios_grid.FIELDS)` is the obvious spelling and it is + # INCOMPLETE: that function's own docstring records that a wall computed from the + # canonical list alone answers "not hidden" for every runtime column, so the + # tenant-wide stratum never reaches the closure and a formula over a hidden column + # comes back. One evaluator, one caller, no second idea of what this session hides. + # ⚠ INSIDE THE `try`, DELIBERATELY. If the wall cannot be computed the route already + # answers with an empty contract rather than a full one — fail-closed is the only + # direction a schema door may fail, and the gate's ADMIN control is what stops that + # silent empty being mistaken for a wall that worked. + hidden = _rc._hidden_for(session) + for f in aios_grid.FIELDS: + if f["key"] in hidden: + continue + entry = {"key": f["key"], "label": f["label"], "type": f["type"], + "source": f["source"], + "description": str(f.get("description") or "")} + if f.get("options"): + entry["options"] = list(f["options"]) + fields.append(entry) + except Exception: + fields = [] + measures = [] + try: + from core import measure_resolve + import core.perm_scope as _perm_scope + team_id = perms.scope_team_id(session.user) + # ⭐⭐ W38-T19 — THE METRICS CAPABILITY REACHES THIS DRAWER TOO, AND IT IS THE ONE DOOR + # THE THREE GRID ASSEMBLIES DO NOT SPEAK FOR. This route calls `measure_resolve.offer` + # ITSELF rather than reading a `measures` key off an assembly, so a wall applied in + # `routes_customers` / `routes_products` / `routes_tables` narrows every grid and leaves + # the panel beside them naming the full lookback catalogue — the same one-door-short + # shape W38-T18 fixed for FIELDS in this very function, one key over. + # ⚠ THE UT_ BRANCH ABOVE RETURNS BEFORE THIS AND ALREADY SERVES `[]`, so there is + # nothing to narrow there; if it ever starts publishing an offer, it needs this line. + _offer = ((measure_resolve.offer(team_id) or []) + if _perm_scope.may_metrics(session.user, key) else []) + for m in _offer: + measures.append({"key": str(m.get("key") or ""), + "label": str(m.get("label") or m.get("key") or ""), + "type": str(m.get("type") or "")}) + except Exception: + measures = [] + out = {"key": key, "label": page.get("label") or key, + "source": page.get("source") or "", "fields": fields, "measures": measures} + if not fields: + # Honest, never a mock: a database whose contract is not yet published says so. + out["note"] = "This database has not published a field contract yet." + return out diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 7795caf496a8598b3c52b4cf5b0663939012d5c8..8e49c41134aafde8a9738e53d63775c2d76fea88 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -1,114 +1,114 @@ -"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). - -Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, -shapes and status codes here; every decision that could be wrong lives in the module a gate -can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, -so the day a second provider lands here is the day nothing in this file changes. - -MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session -`main.py`, and `routes_automation` is already included there — so this router rides inside it -(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that -alters no path. - -⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent -screen** — it is a top-level navigation the client reaches by ``, never JSON. The -callback 302s BACK to the return path the `state` carried (relative-only, sanitised by -`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever -went wrong rides in the query string; a dead-end error page where the app used to be reads as -"the product broke", not "the connect failed". -""" -import os - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import RedirectResponse - -import oauth_connect -from deps import Session, err, require_session - -router = APIRouter(prefix="/oauth") - - -def _redirect_uri(request: Request, provider: str) -> str: - """The redirect URI this deployment registers at the provider — env-pinned when the - container sits behind a proxy that rewrites the scheme (the HF Space), else derived from - the request. MUST match a console-registered URI verbatim, so it is computed in exactly - one place. - - ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to - the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and - the request-derived fallback below is effectively dev-only. - ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host - this returns is where the provider sends the user BACK, and the session cookie is host-only - (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host - the user actually browsed plants the session on the wrong hostname and they return logged - out. Moving the app to a new hostname means moving this value AND re-registering the - resulting URI in the provider console; one without the other fails closed. - Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" - base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") - if not base: - base = f"{request.url.scheme}://{request.url.netloc}" - return f"{base}/api/v1/oauth/{provider}/callback" - - -@router.get("/status") -def oauth_status(session: Session = Depends(require_session)): - """C5's status shape for the SESSION user, one entry per registry provider: - `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's - `ready` reads through.""" - return oauth_connect.status(session.runtime, session.uname) - - -def _offered_or_404(provider: str): - """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. - - The owner pasted the failure this replaces: clicking Connect on the Google card answered - `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is - not offered at all, so the honest status is the one for a URL that does not exist. `404` - rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not - coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). - - ⚠ Every door in this router goes through it, START included, because the JSON the owner saw - came from the start route and a guard on one leg is a guard on one leg. - """ - if not oauth_connect.offered(provider): - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - - -@router.get("/{provider}/start") -def oauth_start(provider: str, request: Request, next: str = "", - session: Session = Depends(require_session)): - """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the - RELATIVE path the callback returns the browser to; it rides inside the single-use state, - sanitised, so the round trip cannot be steered off-origin.""" - _offered_or_404(provider) - url, problem = oauth_connect.start(provider, session.uname, - _redirect_uri(request, provider), next_path=next) - if problem: - raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) - return RedirectResponse(url, status_code=302) - - -@router.get("/{provider}/callback") -def oauth_callback(provider: str, request: Request, - session: Session = Depends(require_session), - state: str = "", code: str = "", error: str = ""): - """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends - the browser back to the state's return path — connected or not (see module header).""" - _offered_or_404(provider) - if error: - home = "/#/" - return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) - email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) - sep = "&" if "?" in home else "?" - if problem: - return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) - return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) - - -@router.post("/{provider}/disconnect") -def oauth_disconnect(provider: str, session: Session = Depends(require_session)): - _offered_or_404(provider) - if oauth_connect.provider_def(provider) is None: - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - oauth_connect.disconnect(session.runtime, session.uname, provider) - return {"disconnected": provider} +"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). + +Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, +shapes and status codes here; every decision that could be wrong lives in the module a gate +can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, +so the day a second provider lands here is the day nothing in this file changes. + +MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session +`main.py`, and `routes_automation` is already included there — so this router rides inside it +(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that +alters no path. + +⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent +screen** — it is a top-level navigation the client reaches by ``, never JSON. The +callback 302s BACK to the return path the `state` carried (relative-only, sanitised by +`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever +went wrong rides in the query string; a dead-end error page where the app used to be reads as +"the product broke", not "the connect failed". +""" +import os + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import RedirectResponse + +import oauth_connect +from deps import Session, err, require_session + +router = APIRouter(prefix="/oauth") + + +def _redirect_uri(request: Request, provider: str) -> str: + """The redirect URI this deployment registers at the provider — env-pinned when the + container sits behind a proxy that rewrites the scheme (the HF Space), else derived from + the request. MUST match a console-registered URI verbatim, so it is computed in exactly + one place. + + ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to + the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and + the request-derived fallback below is effectively dev-only. + ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host + this returns is where the provider sends the user BACK, and the session cookie is host-only + (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host + the user actually browsed plants the session on the wrong hostname and they return logged + out. Moving the app to a new hostname means moving this value AND re-registering the + resulting URI in the provider console; one without the other fails closed. + Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" + base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") + if not base: + base = f"{request.url.scheme}://{request.url.netloc}" + return f"{base}/api/v1/oauth/{provider}/callback" + + +@router.get("/status") +def oauth_status(session: Session = Depends(require_session)): + """C5's status shape for the SESSION user, one entry per registry provider: + `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's + `ready` reads through.""" + return oauth_connect.status(session.runtime, session.uname) + + +def _offered_or_404(provider: str): + """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. + + The owner pasted the failure this replaces: clicking Connect on the Google card answered + `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is + not offered at all, so the honest status is the one for a URL that does not exist. `404` + rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not + coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). + + ⚠ Every door in this router goes through it, START included, because the JSON the owner saw + came from the start route and a guard on one leg is a guard on one leg. + """ + if not oauth_connect.offered(provider): + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + + +@router.get("/{provider}/start") +def oauth_start(provider: str, request: Request, next: str = "", + session: Session = Depends(require_session)): + """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the + RELATIVE path the callback returns the browser to; it rides inside the single-use state, + sanitised, so the round trip cannot be steered off-origin.""" + _offered_or_404(provider) + url, problem = oauth_connect.start(provider, session.uname, + _redirect_uri(request, provider), next_path=next) + if problem: + raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) + return RedirectResponse(url, status_code=302) + + +@router.get("/{provider}/callback") +def oauth_callback(provider: str, request: Request, + session: Session = Depends(require_session), + state: str = "", code: str = "", error: str = ""): + """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends + the browser back to the state's return path — connected or not (see module header).""" + _offered_or_404(provider) + if error: + home = "/#/" + return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) + email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) + sep = "&" if "?" in home else "?" + if problem: + return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) + return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) + + +@router.post("/{provider}/disconnect") +def oauth_disconnect(provider: str, session: Session = Depends(require_session)): + _offered_or_404(provider) + if oauth_connect.provider_def(provider) is None: + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + oauth_connect.disconnect(session.runtime, session.uname, provider) + return {"disconnected": provider} diff --git a/api/routes_products.py b/api/routes_products.py index 93f8a67bab2d58e82364510328e1d646f4f6752a..5c57664e087ba464f57e71fbec329783497748cc 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -433,14 +433,34 @@ def products(session: Session = Depends(module_gate(MODULE))): enforced as a post-filter yields a correct row list carrying both units' numbers. """ import aios_grid + import modules.product_data as pd g = product_assembly(session) rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"]) + # ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a + # report nothing reads is not a report. `product_data.identity_report` is standing rule 1's + # second sentence as data for this grid's two identity columns: `product_id` (the Odoo + # `product.product` id, newly declared in `aios_grid_fields.json`) and `code` (the business + # key, which 33 active products do not have and wear a `pid:` fallback for instead). + # + # ⛔ BUILT OVER `rows_src`, THE WALLED ROWS, NOT OVER THE CATALOGUE. `scoped_pool` runs + # `perm_scope.apply_row_scope` before these pids are taken, so a reader with a permanent + # filter sees fewer products than Odoo has; counting the catalogue here would print 33 beside + # a grid that does not contain 33 such rows, and a number that disagrees with the screen is + # believed anyway. It is the same list `rows_from_pool` was just handed, so the two cannot + # drift. + # + # ⚠ A NEW TOP-LEVEL KEY, NOT A MEMBER OF `identity`. `verify_api`'s W16 section asserts + # `body["identity"] == {"pid": "pid", "businessKey": "code"}` by EQUALITY, so folding the + # report in there would red a gate outside this ticket's fence for a purely cosmetic nesting. + # The envelope check beside it filters to the four `/customers` keys, so an additive key is + # the shape this route already established with `identity` and `scope`. return {"fields": g["fields"], "rows": rows, "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"), "identity": {"pid": "pid", "businessKey": "code"}, + "identity_report": pd.identity_report(g["rows_src"]), "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}} diff --git a/api/routes_publish.py b/api/routes_publish.py index 3c9d85cd01091f4cb99ec7874e1647bf54ba60bd..b0a8bbebf8820fd13b3bc96e8cf175a815430a99 100644 --- a/api/routes_publish.py +++ b/api/routes_publish.py @@ -1,917 +1,917 @@ -"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). - -The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface -(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed -publicly or with a password that the user that share it can toggle."* - -R5, in five clauses, and every one of them is load-bearing: - * publish = a per-view secret token in a **server-only bucket**; - * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; - * an **unauthenticated** read-only route plus a `#/v/` client route; - * the publish surface **projects only the columns the view shows** — never the whole table; - * **creator-or-admin** may publish, and **revoke ROTATES** the token. - -⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door -has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal -for every resolution failure (so the route is not an oracle for which tokens are real), a -constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the -whole body before the handler's first line, and `content-length` is caller-supplied), a sliding -rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. - -⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE -door into a table; a publish token buys a READ projection of one view. Folding them into one index -would make a single leaked string ambiguous about which of the two it opens, and would make -`_resolve` return an object whose capability depends on a field rather than on which door was -knocked. Two buckets, two resolvers, one shape. - -⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and -they are not the same question (audit S-4): the grant registry drives *Shared with me*, -`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing -— an unauthenticated read of a projection, bound to a secret rather than to an account. A published -link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot -be revoked by removing a person, because there is no person. - - python verify_forms.py # this file's gate; section 6 onward -""" -import hashlib -import hmac -import os -import secrets -import time - -from fastapi import APIRouter, Depends, Request - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. -#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec -#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser -#: writes that spec on every autosave — so anything secret living there would be echoed back to the -#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. -TOKENS_KEY = "publish_tokens" -TOKEN_BYTES = 24 - -#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. -#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says -#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is -#: not a stronger string, it is that the secret is never stored, never logged and never echoed. -PW_ITERATIONS = 240_000 -PW_SALT_BYTES = 16 -MIN_PASSPHRASE = 6 -MAX_PASSPHRASE = 128 - -#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so -#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. -RATE_WINDOW_S = 60 -RATE_PER_WINDOW = 30 -MAX_BODY_BYTES = 16 * 1024 - -_HITS: dict = {} - -#: The view kinds that may be published, i.e. R5's "interface". -#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 -#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two -#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a -#: server list that silently drifts from the picker means a mode a user can create and cannot -#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row -#: set; publishing one would be publishing the table, which is exactly what R5's projection clause -#: exists to prevent. -#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. -#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a -#: row set the publisher already curated; a form's row set is other people's answers, gathered under -#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into -#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the -#: product, with no wall between the link and the data. -#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which -#: serves the BLANK form for submitting and never the stored rows. So this is not a capability -#: removed, it is a second door onto the same object that should never have existed beside the -#: first. Publishing a form to be filled in still works, at the URL that was always for it. -#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not -#: offer Publish on a form view, or the picker promises what this refuses. -PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") - - -def _refuse(): - """THE ONE REFUSAL, for every way a published link can fail to resolve. - - Wrong token, revoked link, deleted view, deleted table, a view that stopped being an - interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: - the difference between "no such link" and "that link was revoked" tells an enumerator which - tokens are real, and the difference between "no such link" and "wrong passphrase" tells them - which links are worth guessing at. - """ - return err(403, "bad_publish_token", "that link is not valid") - - -def _same(a: str, b: str) -> bool: - """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" - return hmac.compare_digest(str(a or ""), str(b or "")) - - -def _client_ip(request: Request) -> str: - """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. - - The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new - value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted - through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its - old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the - use it could not support. It is safe as a LOG field and as nothing else. - - The socket peer is the only thing here the caller cannot choose, so it is the key. - ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES - ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm - that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer - is that we cannot separate callers, not that we should throttle them together. - """ - return request.client.host if request.client else "?" - - -def _rate_ok(key: str, now: float) -> bool: - """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. - - A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the - whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire - protection is that guessing a 24-byte token is slow. - - ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer - survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at - all; an enumerator produces nothing but misses. Counting every request under one shared peer - would have denied service to everybody the moment one guesser showed up — trading a token - oracle for an outage is not a fix. - """ - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - _HITS[key] = seen - return len(seen) < RATE_PER_WINDOW - - -def _note_failure(key: str, now: float) -> None: - """Record one failed resolution against this peer, and keep the table bounded.""" - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - seen.append(now) - _HITS[key] = seen - if len(_HITS) > 4096: - for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: - _HITS.pop(k, None) - - -#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash -#: to check against. Its value is irrelevant; its COST is the point. -_DUMMY_SALT = b"\x00" * 16 - - -def _equalise_pw_cost(entry) -> None: - """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. - - MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in - ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording - ("that link is not valid", identical for both) was undone by the clock: anyone could sort real - tokens from fake ones by timing alone, then spend their guesses only on the real ones. The - docstring on `_refuse` describes exactly the leak the code then handed over for free. - - Called on every path that fails BEFORE a passphrase check would have happened, so the cheap - branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so - a link minted under a different iteration count stays indistinguishable too. - """ - iterations = PW_ITERATIONS - if isinstance(entry, dict): - try: - iterations = int(entry.get("iter") or PW_ITERATIONS) - except (TypeError, ValueError): - iterations = PW_ITERATIONS - _hash_pw("", _DUMMY_SALT, iterations) - - -def _public_base() -> str: - return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") - - -def _link_of(token: str) -> str: - return f"{_public_base()}/#/v/{token}" - - -def _tokens(rt) -> dict: - """This tenant's publish index, always a dict.""" - try: - found = rt.get(TOKENS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: - return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), - salt, int(iterations)).hex() - - -def _pw_ok(entry: dict, passphrase: str) -> bool: - """Constant-time verify against the stored digest. - - ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" - from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not - fall open. [[default-must-pass-its-own-guard]] - """ - stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") - if not stored or not salt_hex: - return False - try: - salt = bytes.fromhex(salt_hex) - except ValueError: - return False - got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) - return _same(got, stored) - - -def _find_view(rt, table_key: str, view_id: str): - """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. - - The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` - does, and the reason a view is addressable by id ALONE here: the id is unique within the table, - while the owner is what we are trying to discover. - """ - if not table_key or not view_id: - return None, "" - try: - bucket = rt.get(f"{table_key}_table_workspace") or {} - except Exception: # noqa: BLE001 - return None, "" - if not isinstance(bucket, dict): - return None, "" - for owner, blob in bucket.items(): - views = (blob or {}).get("views") if isinstance(blob, dict) else None - if isinstance(views, dict) and isinstance(views.get(view_id), dict): - return views[view_id], str(owner) - return None, "" - - -def _mode_of(view: dict) -> str: - """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" - cfg = (view or {}).get("config") if isinstance(view, dict) else None - disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None - return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" - - -def _may_administer_view(session: Session, table_key: str, view_id: str): - """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. - - ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such - attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived - as a bare plain-text 500 rather than as our JSON envelope. - """ - import core.table_store as table_store - - if not table_key.startswith("ut_"): - # The Odoo/registry grids are read-through mirrors with their own permission wall; a - # published projection of one would be a second, secret-gated door onto tenant data whose - # visibility the module gate is supposed to decide. - raise err(400, "not_a_database", "you can publish views on your own databases only") - # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING - # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows - # do not live in the tenant document at all — they are windowed out of the DuckDB mirror - # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, - # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a - # token minted here would resolve to a page that could never render. - # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely - # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the - # MINT — where a person is standing in front of the answer — is the only place that reads as a - # sentence rather than as an empty page. - import core.user_tables as user_tables - - if user_tables.is_connected(table_key, st=session.runtime): - raise err(400, "connected_source", - "a database that reads through a connected source (Odoo, Meta Ads) cannot be " - "published as a public link: its rows are served from the tenant's mirror by a " - "signed-in request, and a public link has no session to serve them with") - view, owner = _find_view(session.runtime, table_key, view_id) - if not isinstance(view, dict): - raise err(404, "no_such_view", "that view no longer exists") - if not (owner == session.uname - or table_store._may_administer(view, session.uname, session.admin)): - raise err(403, "not_yours", "only this view's creator or an admin can publish it") - mode = _mode_of(view) - if mode not in PUBLISHABLE_MODES: - raise err(400, "not_an_interface", - "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " - "published as a link") - cfg = view.get("config") or {} - if cfg.get("cohortLock"): - raise err(400, "reader_scoped", - "this view is locked to a cohort, and a cohort's membership is resolved for " - "the person reading it — a public link has no reader, so the page would show " - "no rows at all. Publish a copy without the cohort lock.") - if _needs_a_reader(cfg.get("filters")): - raise err(400, "reader_scoped", - "this view filters on a cohort, a measure rule or a top-N slice, and each of " - "those is resolved for the person reading it — a public link has no reader, so " - "the page would show no rows at all. Publish a copy filtered on columns.") - return view, mode - - -#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not -#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate -#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What -#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than -#: the view has must SAY SO on the wire, with the cause, never just stop. -PUBLIC_ROW_CAP = 5000 - - -def _needs_a_reader(tree) -> bool: - """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? - - ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the - host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an - answer a single ROW cannot compute … absent, the condition matches NOTHING rather than - everything"*. That default is right (it fails closed) and it is unusable here: an anonymous - page whose every row was silently filtered out is indistinguishable from a broken link, and - the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this - repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` - view would serve zero rows on the server while showing twenty in the browser. - - ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. - That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its - cause, never silently enforced. - - ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would - be a second evaluator that agrees today and drifts the day the leaf shape changes — and it - would drift SILENTLY, because the two answers only differ on views nobody has published yet. - [[one-evaluator-per-question]] - """ - from harness import filter_eval - - def walk(node) -> bool: - if isinstance(node, list): - return any(walk(n) for n in node) - if not isinstance(node, dict): - return False - if node.get("kind") == "group" or isinstance(node.get("children"), list): - return any(walk(n) for n in (node.get("children") or [])) - if filter_eval._is_cohort(node) or filter_eval._is_measure(node): - return True - return node.get("op") in filter_eval.RANK_OPS - - return walk(tree) - - -def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): - """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. - - ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a - link dropped the token and left `config.display.published: true` on the view, so the grid's - own UI would go on saying "published" about a link that no longer resolves. Two records of one - fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with - the writer present and the OTHER half forgotten. - - ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render - without asking; they are a MIRROR, never a source, and nothing in this module reads them back - to decide anything. `read_view_link` deliberately reports both so a drift is visible rather - than assumed away. - - ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly - (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything - that is not the literal `public` stores as `password`) — so a value written here and a value - written by the browser cannot disagree. - """ - def _set(cur): - cur = dict(cur or {}) - for owner, blob in list(cur.items()): - views = (blob or {}).get("views") if isinstance(blob, dict) else None - view = views.get(view_id) if isinstance(views, dict) else None - if not isinstance(view, dict): - continue - cfg = dict(view.get("config") or {}) - disp = dict(cfg.get("display") or {}) - if not disp.get("mode"): - # No display block means no interface view; nothing here should invent one. - continue - if published: - disp["published"] = True - disp["publishAccess"] = "public" if access == "public" else "password" - else: - disp.pop("published", None) - disp.pop("publishAccess", None) - cfg["display"] = disp - cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} - return cur - - try: - rt.update(f"{table_key}_table_workspace", _set, flush="sync") - except Exception: # noqa: BLE001 - # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been - # written. A mirror that failed to update leaves the UI one refresh out of date, which is - # strictly better than a 500 on a publish that actually succeeded. - pass - - -def _entry_for(rt, table_key: str, view_id: str): - """`(token, entry)` for a view's existing link, or `(None, None)`.""" - for token, where in _tokens(rt).items(): - if (isinstance(where, dict) and where.get("table") == table_key - and where.get("view") == view_id): - return str(token), where - return None, None - - -def _state_of(token, entry) -> dict: - """The link's state as the SHARER may see it. Built key by key. - - ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire - to the browser, which is the whole failure this file's bucket exists to avoid, and it would do - it silently the first time somebody added a field. - """ - if not token or not isinstance(entry, dict): - return {"published": False, "access": "password", "token": "", "url": ""} - return { - "published": True, - "access": "public" if entry.get("access") == "public" else "password", - "token": str(token), - "url": _link_of(str(token)), - # A boolean, never the digest and never the salt. - "hasPassphrase": bool(entry.get("pw")), - "createdBy": str(entry.get("createdBy") or ""), - } - - -def _resolve(token: str): - """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. - - ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for - THIS view; it does not mean the holder may read anything else, and nothing downstream of here - may widen the subject beyond the `(table, view)` pair the entry names. - - ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the - form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's - string against every stored token in the deployment. - """ - from harness import runtime as _rt - - if not token or len(token) < 16: - return None - for slug in _rt.known_tenants(): - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - for stored, where in _tokens(rt).items(): - if not _same(str(stored), token) or not isinstance(where, dict): - continue - table_key = str(where.get("table") or "") - view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) - # A view deleted, or re-saved as a grid, since the link was minted. Both answer the - # ONE refusal — "that link is not valid" — rather than explaining which. - if not table_key or not isinstance(view, dict): - return None - if _mode_of(view) not in PUBLISHABLE_MODES: - return None - return rt, slug, table_key, view, where - return None - - -def _coord(value, limit: float): - """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. - - ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put - the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of - this code. Anything that is not a finite number inside the earth's range is not a location and - does not travel. - ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` - is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of - Ghana. - """ - if value is None or isinstance(value, bool): - return None - try: - n = float(value) - except (TypeError, ValueError): - return None - return n if n == n and abs(n) <= limit else None - - -def _visible_keys(view: dict, fields: list) -> list: - """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. - - ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` - builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining - field to it — so `order` is every column the table has, hidden ones included. `visible` is the - only allowlist that exists; there is no stored "hidden" key to subtract. - - ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default - set when a saved view carries none, so a public page that read `[]` as an empty allowlist would - render blank — and one that read it as "all fields" would LEAK. The fallback is the same - predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather - than restated here. - """ - import aios_grid - - by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} - stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] - keep = [k for k in stored if k in by_key] - if keep: - return keep - # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. - # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were - # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — - # and the fallback below then WIDENS the public payload to whatever the table declares by - # default. The publisher chose five columns; the anonymous reader gets the table's idea of - # sensible. That is a widening on the one unauthenticated door in the product. - # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` - # at all (a legacy publish, a view saved before the key existed) has no intent to honour and - # the default IS the right answer for it — that is what the fallback was written for. A view - # that stored five keys and has none left DID state an intent, and every column it named is - # gone: the honest answer is no columns, which renders as an empty published view rather than - # somebody else's data. - if stored: - return [] - try: - default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] - except Exception: # noqa: BLE001 - default_visible = [] - fallback = [str(k) for k in default_visible if str(k) in by_key] - return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] - - -def _public_view(rt, table_key: str, view: dict) -> dict: - """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. - - ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, - `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries - every column the table has. Serialising a row and deleting the fields we do not want inverts - the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an - allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the - shipped precedent and says the same thing about itself. - """ - import core.user_tables as user_tables - from harness import filter_eval - - defn = user_tables.get(table_key, st=rt) or {} - fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] - by_key = {str(f.get("key")): f for f in fields} - keys = _visible_keys(view, fields) - cfg = (view or {}).get("config") or {} - - # The row pool: the table's own rows, narrowed to the fields it declares, exactly as - # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has - # no rows here and is refused at the MINT, so this branch is the only one that can be reached. - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - r = {k: v for k, v in (row or {}).items() if k in by_key} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - - # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already - # refused anything this context could not answer, so an empty result here means the filter - # genuinely matches nothing — not that we failed to resolve it. - ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) - keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, - member_pids=cfg.get("memberPids"))) - chosen = [r for r in rows_src if r.get("pid") in keep] - - # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT - # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads - # the way it does; the first fix I wrote was wrong and the second report proved it. - # - # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string - # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` - # projection and the bypass keyed on the FIELD NAME rather than on the value being a - # coordinate. So a column called `lat` could carry anything — a note, an address — and publish - # it. That is the real defect. - # - # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw - # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column - # list — so gating on `visible` meant an ordinary map published a page with no map, under two - # messages that contradicted each other ("no rows carry a location" vs "this view hides its - # location columns"). - # - # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is - # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a - # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a - # VALUE test, so no naming convention can smuggle anything past it. - # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot - # what the server would not send, and the server must not send what the client would discard. - # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here - # because they sit on opposite sides of a trust boundary, where the server's copy is the wall - # and the client's is display hygiene. - mode = _mode_of(view) - plotted = 0 - if mode == "map": - for r in chosen: - if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: - plotted += 1 - - limits = [] - if mode == "map" and chosen and not plotted: - # R6's second sentence. A map with nothing on it must say WHY — and this says the true - # why, which is about the DATA, because visibility is no longer part of the answer. - limits.append({ - "subject": "map", "effect": "not_plotted", - "detail": f"none of these {len(chosen)} rows carry a usable location", - "recommendation": "add `lat` and `lon` values to the records, then reload this link", - }) - if len(chosen) > PUBLIC_ROW_CAP: - # R6's second sentence. A short page that does not say it is short is the silent - # truncation the rule is actually about. - limits.append({ - "subject": "rows", "effect": "windowed", - "detail": f"this view has {len(chosen)} rows and a published page serves the first " - f"{PUBLIC_ROW_CAP}", - "recommendation": "narrow the view's filters, or share it with named people instead " - "of publishing a link", - }) - chosen = chosen[:PUBLIC_ROW_CAP] - - # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours - # by a column the view HIDES would otherwise put that column's value on every public row — - # the projection leak, arriving through the renderer rather than through the column list. The - # fail-closed choice is to drop the ref and render the map without colour; a published page - # that is slightly plainer beats one that ships a hidden column. - disp_in = (cfg.get("display") or {}) - display = {"mode": mode} - for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): - if disp_in.get(ref) in keys: - display[ref] = disp_in[ref] - - return { - "title": str((view or {}).get("name") or "")[:200], - "mode": display["mode"], - "display": display, - "columns": [{"key": k, - "label": str(by_key[k].get("label") or k), - "type": str(by_key[k].get("type") or "text"), - **({"options": [str(o) for o in by_key[k]["options"]][:200]} - if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") - else {})} - for k in keys], - # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; - # it is a row NUMBER within this table and names nothing outside it. - # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second - # half was missing, which was a LEAK. Found by a verifier that drove this route with a - # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out - # on the wire. - # - # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never - # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds - # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives - # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a - # view can hide them like any other. Hiding them therefore has to work here, because on a - # published page **the projection is the only wall there is** (`ut_*` databases have no - # hidden-field closure behind it, `routes_shares.py`'s docstring). - # - # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it - # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. - # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not - # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the - # names while both values were `None` [[gate-answers-the-wrong-question]]. - # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row - # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, - # and only when they PARSE as coordinates — see the block above for why that is the test. - "rows": [{"pid": r.get("pid"), - **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} - if mode == "map" - and _coord(r.get("lat"), 90) is not None - and _coord(r.get("lon"), 180) is not None else {}), - **{k: r.get(k) for k in keys}} for r in chosen], - "total": len(chosen), - **({"limits": limits} if limits else {}), - } - - -async def _bounded_body(request: Request) -> dict: - """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. - - FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a - declared model is not a bound at all; `content-length` is caller-supplied, so checking it is - not one either. Streaming with a running total is the actual bound. - """ - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > MAX_BODY_BYTES: - raise err(413, "body_too_large", "that request is too large") - chunks.append(chunk) - import json - try: - parsed = json.loads(b"".join(chunks) or b"{}") - except ValueError: - raise err(400, "bad_request", "that request could not be read") - return parsed if isinstance(parsed, dict) else {} - - -# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── -# -# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form -# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` -# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no -# order to get wrong. - - -@router.get("/view-link") -def read_view_link(topic: str = "", view: str = "", - session: Session = Depends(require_session)): - """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" - v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) - token, entry = _entry_for(session.runtime, str(topic), str(view)) - out = _state_of(token, entry) - # The presentational flags contract C1 put on the view spec, echoed back so the client can - # tell whether the two agree. They are a MIRROR of this bucket, never its source. - disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} - out["displayPublished"] = disp.get("published") is True - return out - - -@router.post("/view-link") -async def mint_view_link(request: Request, session: Session = Depends(require_session)): - """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. - - IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not - invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous - one dies in the SAME write, so there is never a window where both open the view. - """ - body = await _bounded_body(request) - topic, view = str(body.get("topic") or ""), str(body.get("view") or "") - _v, _mode = _may_administer_view(session, topic, view) - - access = "public" if body.get("access") == "public" else "password" - raw_pw = body.get("passphrase") - passphrase = "" if raw_pw is None else str(raw_pw) - if len(passphrase) > MAX_PASSPHRASE: - raise err(400, "passphrase_too_long", - f"a passphrase can be at most {MAX_PASSPHRASE} characters") - - existing_token, existing = _entry_for(session.runtime, topic, view) - rotate = bool(body.get("rotate")) - fresh = secrets.token_urlsafe(TOKEN_BYTES) - - # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: - # the caller supplied a passphrase now, or one was already stored and is being kept. Anything - # else is refused HERE rather than stored and refused later — a link that cannot be opened by - # anybody is not a safe default, it is a broken feature that reads as a permission bug. - if access == "password": - if passphrase and len(passphrase) < MIN_PASSPHRASE: - raise err(400, "passphrase_too_short", - f"a passphrase needs at least {MIN_PASSPHRASE} characters") - if not passphrase and not (existing or {}).get("pw"): - raise err(400, "passphrase_required", - "a password-protected link needs a passphrase") - - minted = {"token": existing_token or fresh} - - def _set(cur): - cur = dict(cur or {}) - prev = None - for stored, where in list(cur.items()): - if (isinstance(where, dict) and where.get("table") == topic - and where.get("view") == view): - prev = dict(where) - cur.pop(stored, None) - if not rotate: - minted["token"] = str(stored) - if rotate or prev is None: - minted["token"] = fresh - entry = {"table": topic, "view": view, "access": access, - "createdBy": str((prev or {}).get("createdBy") or session.uname), - "createdAt": float((prev or {}).get("createdAt") or time.time())} - # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED - # it — found by a verifier that traced the toggle rather than the happy path. Publishing - # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was - # kept; switching back to `password` then demanded a new passphrase, silently, while the - # panel's own sentence promised the opposite ("leave blank to keep the current one"). - # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` - # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, - # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI - # makes and the store breaks is a defect either way, and the honest fix is to keep the - # promise rather than to reword it. - if passphrase: - salt = secrets.token_bytes(PW_SALT_BYTES) - entry["salt"] = salt.hex() - entry["iter"] = PW_ITERATIONS - entry["pw"] = _hash_pw(passphrase, salt) - elif (prev or {}).get("pw"): - # Carried key by key, so a future field on the entry is not silently inherited. - entry["salt"] = str((prev or {}).get("salt") or "") - entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) - entry["pw"] = str((prev or {}).get("pw") or "") - cur[minted["token"]] = entry - return cur - - session.runtime.update(TOKENS_KEY, _set, flush="sync") - _mirror_display(session.runtime, topic, view, True, access) - token, entry = _entry_for(session.runtime, topic, view) - # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered - # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that - # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back - # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. - # If the write cannot be read back, say so; do not describe a state nobody verified. - if not entry: - raise err(503, "not_saved", - "the link was minted but could not be read back — reload and try again") - return _state_of(token, entry) - - -@router.delete("/view-link") -async def revoke_view_link(request: Request, session: Session = Depends(require_session)): - """Unpublish. `{topic, view}`. - - ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the - index in this write, so the old link stops resolving immediately; and because a later publish - mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An - implementation that kept the token and flipped an `enabled` flag would leave the secret live - in the store, one bug away from working again, and would make "revoked" a property somebody - could forget to check on a code path added later. - """ - body = await _bounded_body(request) - topic, view = str(body.get("topic") or ""), str(body.get("view") or "") - _may_administer_view(session, topic, view) - - def _drop(cur): - cur = dict(cur or {}) - for stored, where in list(cur.items()): - if (isinstance(where, dict) and where.get("table") == topic - and where.get("view") == view): - cur.pop(stored, None) - return cur - - session.runtime.update(TOKENS_KEY, _drop, flush="sync") - # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing - # "published" about a link that no longer resolves — the half a reviewer caught. - _mirror_display(session.runtime, topic, view, False) - return {"published": False, "access": "password", "token": "", "url": ""} - - -# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── -# -# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not -# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a -# route is public exactly by omitting the dependency. Which is why the two are kept together, -# under one banner, rather than filed beside the sharer's routes they resemble. - - -@router.get("/published/{token}") -def get_published(token: str, request: Request): - """The published view. Read-only, unauthenticated, projected to the view's own columns.""" - if not _rate_ok(_client_ip(request), time.time()): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - found = _resolve(token) - if not found: - # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. - # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, - # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own - # docstring says the difference between "no such link" and "that link was revoked" must - # never be observable — and the route beside it published that difference in its status - # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the - # passphrase is all that is left and it can be attacked offline-cheap. - # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no - # title, no columns, no count, the same bytes a real password link returns before anyone - # has tried to open it. The guess then costs a POST with a passphrase, which is rate - # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public - # link is for; what stops being visible is which PASSWORD tokens exist. - _note_failure(_client_ip(request), time.time()) - _equalise_pw_cost(None) - return {"locked": True} - rt, _slug, table_key, view, entry = found - if entry.get("access") == "password": - # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render - # a passphrase prompt, so this 200 says "there is something here and it is locked" and - # NOTHING else — no title, no column names, no row count. A 403 here would be - # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for - # a link the holder has not tried to open yet. - return {"locked": True} - return _public_view(rt, table_key, view) - - -@router.post("/published/{token}") -async def open_published(token: str, request: Request): - """Open a password-protected link. `{passphrase}`. - - ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they - differed, the route would confirm which tokens are real to anybody willing to send one guess — - and a 24-byte token's entire protection is that it cannot be found by guessing. - ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never - stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and - anyone may claim an identity; this is AUTHENTICATION and is treated as one. - """ - now = time.time() - if not _rate_ok(_client_ip(request), now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - body = await _bounded_body(request) - found = _resolve(token) - if not found: - # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and - # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 - # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. - # Both arms now cost the same, so the identical 403 above is finally identical in practice - # rather than only in wording. - _note_failure(_client_ip(request), now) - _equalise_pw_cost(None) - raise _refuse() - rt, _slug, table_key, view, entry = found - if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): - _note_failure(_client_ip(request), now) - raise _refuse() - # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the - # clock on a route whose whole job is to be uninformative. - if entry.get("access") != "password": - _equalise_pw_cost(entry) - return _public_view(rt, table_key, view) +"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). + +The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface +(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed +publicly or with a password that the user that share it can toggle."* + +R5, in five clauses, and every one of them is load-bearing: + * publish = a per-view secret token in a **server-only bucket**; + * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; + * an **unauthenticated** read-only route plus a `#/v/` client route; + * the publish surface **projects only the columns the view shows** — never the whole table; + * **creator-or-admin** may publish, and **revoke ROTATES** the token. + +⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door +has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal +for every resolution failure (so the route is not an oracle for which tokens are real), a +constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the +whole body before the handler's first line, and `content-length` is caller-supplied), a sliding +rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. + +⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE +door into a table; a publish token buys a READ projection of one view. Folding them into one index +would make a single leaked string ambiguous about which of the two it opens, and would make +`_resolve` return an object whose capability depends on a field rather than on which door was +knocked. Two buckets, two resolvers, one shape. + +⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and +they are not the same question (audit S-4): the grant registry drives *Shared with me*, +`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing +— an unauthenticated read of a projection, bound to a secret rather than to an account. A published +link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot +be revoked by removing a person, because there is no person. + + python verify_forms.py # this file's gate; section 6 onward +""" +import hashlib +import hmac +import os +import secrets +import time + +from fastapi import APIRouter, Depends, Request + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. +#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec +#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser +#: writes that spec on every autosave — so anything secret living there would be echoed back to the +#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. +TOKENS_KEY = "publish_tokens" +TOKEN_BYTES = 24 + +#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. +#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says +#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is +#: not a stronger string, it is that the secret is never stored, never logged and never echoed. +PW_ITERATIONS = 240_000 +PW_SALT_BYTES = 16 +MIN_PASSPHRASE = 6 +MAX_PASSPHRASE = 128 + +#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so +#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. +RATE_WINDOW_S = 60 +RATE_PER_WINDOW = 30 +MAX_BODY_BYTES = 16 * 1024 + +_HITS: dict = {} + +#: The view kinds that may be published, i.e. R5's "interface". +#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 +#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two +#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a +#: server list that silently drifts from the picker means a mode a user can create and cannot +#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row +#: set; publishing one would be publishing the table, which is exactly what R5's projection clause +#: exists to prevent. +#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. +#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a +#: row set the publisher already curated; a form's row set is other people's answers, gathered under +#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into +#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the +#: product, with no wall between the link and the data. +#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which +#: serves the BLANK form for submitting and never the stored rows. So this is not a capability +#: removed, it is a second door onto the same object that should never have existed beside the +#: first. Publishing a form to be filled in still works, at the URL that was always for it. +#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not +#: offer Publish on a form view, or the picker promises what this refuses. +PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") + + +def _refuse(): + """THE ONE REFUSAL, for every way a published link can fail to resolve. + + Wrong token, revoked link, deleted view, deleted table, a view that stopped being an + interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: + the difference between "no such link" and "that link was revoked" tells an enumerator which + tokens are real, and the difference between "no such link" and "wrong passphrase" tells them + which links are worth guessing at. + """ + return err(403, "bad_publish_token", "that link is not valid") + + +def _same(a: str, b: str) -> bool: + """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" + return hmac.compare_digest(str(a or ""), str(b or "")) + + +def _client_ip(request: Request) -> str: + """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. + + The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new + value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted + through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its + old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the + use it could not support. It is safe as a LOG field and as nothing else. + + The socket peer is the only thing here the caller cannot choose, so it is the key. + ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES + ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm + that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer + is that we cannot separate callers, not that we should throttle them together. + """ + return request.client.host if request.client else "?" + + +def _rate_ok(key: str, now: float) -> bool: + """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. + + A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the + whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire + protection is that guessing a 24-byte token is slow. + + ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer + survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at + all; an enumerator produces nothing but misses. Counting every request under one shared peer + would have denied service to everybody the moment one guesser showed up — trading a token + oracle for an outage is not a fix. + """ + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + _HITS[key] = seen + return len(seen) < RATE_PER_WINDOW + + +def _note_failure(key: str, now: float) -> None: + """Record one failed resolution against this peer, and keep the table bounded.""" + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + + +#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash +#: to check against. Its value is irrelevant; its COST is the point. +_DUMMY_SALT = b"\x00" * 16 + + +def _equalise_pw_cost(entry) -> None: + """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. + + MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in + ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording + ("that link is not valid", identical for both) was undone by the clock: anyone could sort real + tokens from fake ones by timing alone, then spend their guesses only on the real ones. The + docstring on `_refuse` describes exactly the leak the code then handed over for free. + + Called on every path that fails BEFORE a passphrase check would have happened, so the cheap + branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so + a link minted under a different iteration count stays indistinguishable too. + """ + iterations = PW_ITERATIONS + if isinstance(entry, dict): + try: + iterations = int(entry.get("iter") or PW_ITERATIONS) + except (TypeError, ValueError): + iterations = PW_ITERATIONS + _hash_pw("", _DUMMY_SALT, iterations) + + +def _public_base() -> str: + return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") + + +def _link_of(token: str) -> str: + return f"{_public_base()}/#/v/{token}" + + +def _tokens(rt) -> dict: + """This tenant's publish index, always a dict.""" + try: + found = rt.get(TOKENS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: + return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), + salt, int(iterations)).hex() + + +def _pw_ok(entry: dict, passphrase: str) -> bool: + """Constant-time verify against the stored digest. + + ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" + from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not + fall open. [[default-must-pass-its-own-guard]] + """ + stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") + if not stored or not salt_hex: + return False + try: + salt = bytes.fromhex(salt_hex) + except ValueError: + return False + got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) + return _same(got, stored) + + +def _find_view(rt, table_key: str, view_id: str): + """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. + + The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` + does, and the reason a view is addressable by id ALONE here: the id is unique within the table, + while the owner is what we are trying to discover. + """ + if not table_key or not view_id: + return None, "" + try: + bucket = rt.get(f"{table_key}_table_workspace") or {} + except Exception: # noqa: BLE001 + return None, "" + if not isinstance(bucket, dict): + return None, "" + for owner, blob in bucket.items(): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + if isinstance(views, dict) and isinstance(views.get(view_id), dict): + return views[view_id], str(owner) + return None, "" + + +def _mode_of(view: dict) -> str: + """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" + cfg = (view or {}).get("config") if isinstance(view, dict) else None + disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None + return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" + + +def _may_administer_view(session: Session, table_key: str, view_id: str): + """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. + + ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such + attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived + as a bare plain-text 500 rather than as our JSON envelope. + """ + import core.table_store as table_store + + if not table_key.startswith("ut_"): + # The Odoo/registry grids are read-through mirrors with their own permission wall; a + # published projection of one would be a second, secret-gated door onto tenant data whose + # visibility the module gate is supposed to decide. + raise err(400, "not_a_database", "you can publish views on your own databases only") + # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING + # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows + # do not live in the tenant document at all — they are windowed out of the DuckDB mirror + # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, + # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a + # token minted here would resolve to a page that could never render. + # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely + # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the + # MINT — where a person is standing in front of the answer — is the only place that reads as a + # sentence rather than as an empty page. + import core.user_tables as user_tables + + if user_tables.is_connected(table_key, st=session.runtime): + raise err(400, "connected_source", + "a database that reads through a connected source (Odoo, Meta Ads) cannot be " + "published as a public link: its rows are served from the tenant's mirror by a " + "signed-in request, and a public link has no session to serve them with") + view, owner = _find_view(session.runtime, table_key, view_id) + if not isinstance(view, dict): + raise err(404, "no_such_view", "that view no longer exists") + if not (owner == session.uname + or table_store._may_administer(view, session.uname, session.admin)): + raise err(403, "not_yours", "only this view's creator or an admin can publish it") + mode = _mode_of(view) + if mode not in PUBLISHABLE_MODES: + raise err(400, "not_an_interface", + "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " + "published as a link") + cfg = view.get("config") or {} + if cfg.get("cohortLock"): + raise err(400, "reader_scoped", + "this view is locked to a cohort, and a cohort's membership is resolved for " + "the person reading it — a public link has no reader, so the page would show " + "no rows at all. Publish a copy without the cohort lock.") + if _needs_a_reader(cfg.get("filters")): + raise err(400, "reader_scoped", + "this view filters on a cohort, a measure rule or a top-N slice, and each of " + "those is resolved for the person reading it — a public link has no reader, so " + "the page would show no rows at all. Publish a copy filtered on columns.") + return view, mode + + +#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not +#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate +#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What +#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than +#: the view has must SAY SO on the wire, with the cause, never just stop. +PUBLIC_ROW_CAP = 5000 + + +def _needs_a_reader(tree) -> bool: + """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? + + ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the + host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an + answer a single ROW cannot compute … absent, the condition matches NOTHING rather than + everything"*. That default is right (it fails closed) and it is unusable here: an anonymous + page whose every row was silently filtered out is indistinguishable from a broken link, and + the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this + repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` + view would serve zero rows on the server while showing twenty in the browser. + + ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. + That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its + cause, never silently enforced. + + ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would + be a second evaluator that agrees today and drifts the day the leaf shape changes — and it + would drift SILENTLY, because the two answers only differ on views nobody has published yet. + [[one-evaluator-per-question]] + """ + from harness import filter_eval + + def walk(node) -> bool: + if isinstance(node, list): + return any(walk(n) for n in node) + if not isinstance(node, dict): + return False + if node.get("kind") == "group" or isinstance(node.get("children"), list): + return any(walk(n) for n in (node.get("children") or [])) + if filter_eval._is_cohort(node) or filter_eval._is_measure(node): + return True + return node.get("op") in filter_eval.RANK_OPS + + return walk(tree) + + +def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): + """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. + + ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a + link dropped the token and left `config.display.published: true` on the view, so the grid's + own UI would go on saying "published" about a link that no longer resolves. Two records of one + fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with + the writer present and the OTHER half forgotten. + + ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render + without asking; they are a MIRROR, never a source, and nothing in this module reads them back + to decide anything. `read_view_link` deliberately reports both so a drift is visible rather + than assumed away. + + ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly + (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything + that is not the literal `public` stores as `password`) — so a value written here and a value + written by the browser cannot disagree. + """ + def _set(cur): + cur = dict(cur or {}) + for owner, blob in list(cur.items()): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + view = views.get(view_id) if isinstance(views, dict) else None + if not isinstance(view, dict): + continue + cfg = dict(view.get("config") or {}) + disp = dict(cfg.get("display") or {}) + if not disp.get("mode"): + # No display block means no interface view; nothing here should invent one. + continue + if published: + disp["published"] = True + disp["publishAccess"] = "public" if access == "public" else "password" + else: + disp.pop("published", None) + disp.pop("publishAccess", None) + cfg["display"] = disp + cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} + return cur + + try: + rt.update(f"{table_key}_table_workspace", _set, flush="sync") + except Exception: # noqa: BLE001 + # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been + # written. A mirror that failed to update leaves the UI one refresh out of date, which is + # strictly better than a 500 on a publish that actually succeeded. + pass + + +def _entry_for(rt, table_key: str, view_id: str): + """`(token, entry)` for a view's existing link, or `(None, None)`.""" + for token, where in _tokens(rt).items(): + if (isinstance(where, dict) and where.get("table") == table_key + and where.get("view") == view_id): + return str(token), where + return None, None + + +def _state_of(token, entry) -> dict: + """The link's state as the SHARER may see it. Built key by key. + + ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire + to the browser, which is the whole failure this file's bucket exists to avoid, and it would do + it silently the first time somebody added a field. + """ + if not token or not isinstance(entry, dict): + return {"published": False, "access": "password", "token": "", "url": ""} + return { + "published": True, + "access": "public" if entry.get("access") == "public" else "password", + "token": str(token), + "url": _link_of(str(token)), + # A boolean, never the digest and never the salt. + "hasPassphrase": bool(entry.get("pw")), + "createdBy": str(entry.get("createdBy") or ""), + } + + +def _resolve(token: str): + """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. + + ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for + THIS view; it does not mean the holder may read anything else, and nothing downstream of here + may widen the subject beyond the `(table, view)` pair the entry names. + + ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the + form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's + string against every stored token in the deployment. + """ + from harness import runtime as _rt + + if not token or len(token) < 16: + return None + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + for stored, where in _tokens(rt).items(): + if not _same(str(stored), token) or not isinstance(where, dict): + continue + table_key = str(where.get("table") or "") + view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) + # A view deleted, or re-saved as a grid, since the link was minted. Both answer the + # ONE refusal — "that link is not valid" — rather than explaining which. + if not table_key or not isinstance(view, dict): + return None + if _mode_of(view) not in PUBLISHABLE_MODES: + return None + return rt, slug, table_key, view, where + return None + + +def _coord(value, limit: float): + """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. + + ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put + the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of + this code. Anything that is not a finite number inside the earth's range is not a location and + does not travel. + ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` + is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of + Ghana. + """ + if value is None or isinstance(value, bool): + return None + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if n == n and abs(n) <= limit else None + + +def _visible_keys(view: dict, fields: list) -> list: + """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. + + ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` + builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining + field to it — so `order` is every column the table has, hidden ones included. `visible` is the + only allowlist that exists; there is no stored "hidden" key to subtract. + + ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default + set when a saved view carries none, so a public page that read `[]` as an empty allowlist would + render blank — and one that read it as "all fields" would LEAK. The fallback is the same + predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather + than restated here. + """ + import aios_grid + + by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} + stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] + keep = [k for k in stored if k in by_key] + if keep: + return keep + # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. + # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were + # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — + # and the fallback below then WIDENS the public payload to whatever the table declares by + # default. The publisher chose five columns; the anonymous reader gets the table's idea of + # sensible. That is a widening on the one unauthenticated door in the product. + # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` + # at all (a legacy publish, a view saved before the key existed) has no intent to honour and + # the default IS the right answer for it — that is what the fallback was written for. A view + # that stored five keys and has none left DID state an intent, and every column it named is + # gone: the honest answer is no columns, which renders as an empty published view rather than + # somebody else's data. + if stored: + return [] + try: + default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] + except Exception: # noqa: BLE001 + default_visible = [] + fallback = [str(k) for k in default_visible if str(k) in by_key] + return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] + + +def _public_view(rt, table_key: str, view: dict) -> dict: + """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. + + ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, + `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries + every column the table has. Serialising a row and deleting the fields we do not want inverts + the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an + allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the + shipped precedent and says the same thing about itself. + """ + import core.user_tables as user_tables + from harness import filter_eval + + defn = user_tables.get(table_key, st=rt) or {} + fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] + by_key = {str(f.get("key")): f for f in fields} + keys = _visible_keys(view, fields) + cfg = (view or {}).get("config") or {} + + # The row pool: the table's own rows, narrowed to the fields it declares, exactly as + # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has + # no rows here and is refused at the MINT, so this branch is the only one that can be reached. + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in by_key} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + + # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already + # refused anything this context could not answer, so an empty result here means the filter + # genuinely matches nothing — not that we failed to resolve it. + ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) + keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, + member_pids=cfg.get("memberPids"))) + chosen = [r for r in rows_src if r.get("pid") in keep] + + # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT + # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads + # the way it does; the first fix I wrote was wrong and the second report proved it. + # + # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string + # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` + # projection and the bypass keyed on the FIELD NAME rather than on the value being a + # coordinate. So a column called `lat` could carry anything — a note, an address — and publish + # it. That is the real defect. + # + # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw + # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column + # list — so gating on `visible` meant an ordinary map published a page with no map, under two + # messages that contradicted each other ("no rows carry a location" vs "this view hides its + # location columns"). + # + # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is + # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a + # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a + # VALUE test, so no naming convention can smuggle anything past it. + # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot + # what the server would not send, and the server must not send what the client would discard. + # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here + # because they sit on opposite sides of a trust boundary, where the server's copy is the wall + # and the client's is display hygiene. + mode = _mode_of(view) + plotted = 0 + if mode == "map": + for r in chosen: + if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: + plotted += 1 + + limits = [] + if mode == "map" and chosen and not plotted: + # R6's second sentence. A map with nothing on it must say WHY — and this says the true + # why, which is about the DATA, because visibility is no longer part of the answer. + limits.append({ + "subject": "map", "effect": "not_plotted", + "detail": f"none of these {len(chosen)} rows carry a usable location", + "recommendation": "add `lat` and `lon` values to the records, then reload this link", + }) + if len(chosen) > PUBLIC_ROW_CAP: + # R6's second sentence. A short page that does not say it is short is the silent + # truncation the rule is actually about. + limits.append({ + "subject": "rows", "effect": "windowed", + "detail": f"this view has {len(chosen)} rows and a published page serves the first " + f"{PUBLIC_ROW_CAP}", + "recommendation": "narrow the view's filters, or share it with named people instead " + "of publishing a link", + }) + chosen = chosen[:PUBLIC_ROW_CAP] + + # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours + # by a column the view HIDES would otherwise put that column's value on every public row — + # the projection leak, arriving through the renderer rather than through the column list. The + # fail-closed choice is to drop the ref and render the map without colour; a published page + # that is slightly plainer beats one that ships a hidden column. + disp_in = (cfg.get("display") or {}) + display = {"mode": mode} + for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): + if disp_in.get(ref) in keys: + display[ref] = disp_in[ref] + + return { + "title": str((view or {}).get("name") or "")[:200], + "mode": display["mode"], + "display": display, + "columns": [{"key": k, + "label": str(by_key[k].get("label") or k), + "type": str(by_key[k].get("type") or "text"), + **({"options": [str(o) for o in by_key[k]["options"]][:200]} + if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") + else {})} + for k in keys], + # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; + # it is a row NUMBER within this table and names nothing outside it. + # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second + # half was missing, which was a LEAK. Found by a verifier that drove this route with a + # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out + # on the wire. + # + # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never + # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds + # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives + # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a + # view can hide them like any other. Hiding them therefore has to work here, because on a + # published page **the projection is the only wall there is** (`ut_*` databases have no + # hidden-field closure behind it, `routes_shares.py`'s docstring). + # + # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it + # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. + # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not + # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the + # names while both values were `None` [[gate-answers-the-wrong-question]]. + # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row + # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, + # and only when they PARSE as coordinates — see the block above for why that is the test. + "rows": [{"pid": r.get("pid"), + **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} + if mode == "map" + and _coord(r.get("lat"), 90) is not None + and _coord(r.get("lon"), 180) is not None else {}), + **{k: r.get(k) for k in keys}} for r in chosen], + "total": len(chosen), + **({"limits": limits} if limits else {}), + } + + +async def _bounded_body(request: Request) -> dict: + """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. + + FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a + declared model is not a bound at all; `content-length` is caller-supplied, so checking it is + not one either. Streaming with a running total is the actual bound. + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + import json + try: + parsed = json.loads(b"".join(chunks) or b"{}") + except ValueError: + raise err(400, "bad_request", "that request could not be read") + return parsed if isinstance(parsed, dict) else {} + + +# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── +# +# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form +# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` +# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no +# order to get wrong. + + +@router.get("/view-link") +def read_view_link(topic: str = "", view: str = "", + session: Session = Depends(require_session)): + """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" + v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) + token, entry = _entry_for(session.runtime, str(topic), str(view)) + out = _state_of(token, entry) + # The presentational flags contract C1 put on the view spec, echoed back so the client can + # tell whether the two agree. They are a MIRROR of this bucket, never its source. + disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} + out["displayPublished"] = disp.get("published") is True + return out + + +@router.post("/view-link") +async def mint_view_link(request: Request, session: Session = Depends(require_session)): + """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. + + IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not + invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous + one dies in the SAME write, so there is never a window where both open the view. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _v, _mode = _may_administer_view(session, topic, view) + + access = "public" if body.get("access") == "public" else "password" + raw_pw = body.get("passphrase") + passphrase = "" if raw_pw is None else str(raw_pw) + if len(passphrase) > MAX_PASSPHRASE: + raise err(400, "passphrase_too_long", + f"a passphrase can be at most {MAX_PASSPHRASE} characters") + + existing_token, existing = _entry_for(session.runtime, topic, view) + rotate = bool(body.get("rotate")) + fresh = secrets.token_urlsafe(TOKEN_BYTES) + + # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: + # the caller supplied a passphrase now, or one was already stored and is being kept. Anything + # else is refused HERE rather than stored and refused later — a link that cannot be opened by + # anybody is not a safe default, it is a broken feature that reads as a permission bug. + if access == "password": + if passphrase and len(passphrase) < MIN_PASSPHRASE: + raise err(400, "passphrase_too_short", + f"a passphrase needs at least {MIN_PASSPHRASE} characters") + if not passphrase and not (existing or {}).get("pw"): + raise err(400, "passphrase_required", + "a password-protected link needs a passphrase") + + minted = {"token": existing_token or fresh} + + def _set(cur): + cur = dict(cur or {}) + prev = None + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + prev = dict(where) + cur.pop(stored, None) + if not rotate: + minted["token"] = str(stored) + if rotate or prev is None: + minted["token"] = fresh + entry = {"table": topic, "view": view, "access": access, + "createdBy": str((prev or {}).get("createdBy") or session.uname), + "createdAt": float((prev or {}).get("createdAt") or time.time())} + # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED + # it — found by a verifier that traced the toggle rather than the happy path. Publishing + # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was + # kept; switching back to `password` then demanded a new passphrase, silently, while the + # panel's own sentence promised the opposite ("leave blank to keep the current one"). + # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` + # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, + # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI + # makes and the store breaks is a defect either way, and the honest fix is to keep the + # promise rather than to reword it. + if passphrase: + salt = secrets.token_bytes(PW_SALT_BYTES) + entry["salt"] = salt.hex() + entry["iter"] = PW_ITERATIONS + entry["pw"] = _hash_pw(passphrase, salt) + elif (prev or {}).get("pw"): + # Carried key by key, so a future field on the entry is not silently inherited. + entry["salt"] = str((prev or {}).get("salt") or "") + entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) + entry["pw"] = str((prev or {}).get("pw") or "") + cur[minted["token"]] = entry + return cur + + session.runtime.update(TOKENS_KEY, _set, flush="sync") + _mirror_display(session.runtime, topic, view, True, access) + token, entry = _entry_for(session.runtime, topic, view) + # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered + # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that + # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back + # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. + # If the write cannot be read back, say so; do not describe a state nobody verified. + if not entry: + raise err(503, "not_saved", + "the link was minted but could not be read back — reload and try again") + return _state_of(token, entry) + + +@router.delete("/view-link") +async def revoke_view_link(request: Request, session: Session = Depends(require_session)): + """Unpublish. `{topic, view}`. + + ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the + index in this write, so the old link stops resolving immediately; and because a later publish + mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An + implementation that kept the token and flipped an `enabled` flag would leave the secret live + in the store, one bug away from working again, and would make "revoked" a property somebody + could forget to check on a code path added later. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _may_administer_view(session, topic, view) + + def _drop(cur): + cur = dict(cur or {}) + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + cur.pop(stored, None) + return cur + + session.runtime.update(TOKENS_KEY, _drop, flush="sync") + # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing + # "published" about a link that no longer resolves — the half a reviewer caught. + _mirror_display(session.runtime, topic, view, False) + return {"published": False, "access": "password", "token": "", "url": ""} + + +# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── +# +# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not +# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a +# route is public exactly by omitting the dependency. Which is why the two are kept together, +# under one banner, rather than filed beside the sharer's routes they resemble. + + +@router.get("/published/{token}") +def get_published(token: str, request: Request): + """The published view. Read-only, unauthenticated, projected to the view's own columns.""" + if not _rate_ok(_client_ip(request), time.time()): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. + # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, + # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own + # docstring says the difference between "no such link" and "that link was revoked" must + # never be observable — and the route beside it published that difference in its status + # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the + # passphrase is all that is left and it can be attacked offline-cheap. + # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no + # title, no columns, no count, the same bytes a real password link returns before anyone + # has tried to open it. The guess then costs a POST with a passphrase, which is rate + # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public + # link is for; what stops being visible is which PASSWORD tokens exist. + _note_failure(_client_ip(request), time.time()) + _equalise_pw_cost(None) + return {"locked": True} + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password": + # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render + # a passphrase prompt, so this 200 says "there is something here and it is locked" and + # NOTHING else — no title, no column names, no row count. A 403 here would be + # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for + # a link the holder has not tried to open yet. + return {"locked": True} + return _public_view(rt, table_key, view) + + +@router.post("/published/{token}") +async def open_published(token: str, request: Request): + """Open a password-protected link. `{passphrase}`. + + ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they + differed, the route would confirm which tokens are real to anybody willing to send one guess — + and a 24-byte token's entire protection is that it cannot be found by guessing. + ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never + stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and + anyone may claim an identity; this is AUTHENTICATION and is treated as one. + """ + now = time.time() + if not _rate_ok(_client_ip(request), now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + body = await _bounded_body(request) + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and + # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 + # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. + # Both arms now cost the same, so the identical 403 above is finally identical in practice + # rather than only in wording. + _note_failure(_client_ip(request), now) + _equalise_pw_cost(None) + raise _refuse() + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): + _note_failure(_client_ip(request), now) + raise _refuse() + # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the + # clock on a route whose whole job is to be uninformative. + if entry.get("access") != "password": + _equalise_pw_cost(entry) + return _public_view(rt, table_key, view) diff --git a/api/routes_records.py b/api/routes_records.py index c37f74b261d63bb390db45660363e8d94ff796d4..ef0bb2c8153db1e40b2656a715f545d80b1fe22b 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -1,211 +1,211 @@ -"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. - -⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a -customer-shaped wall bolted to the module import line: `_in_book` asked -`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and -typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the -panel answered "that customer is not in your book" — the owner's report. The dangerous half is -the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment -is filed against somebody's customer where the whole team can read it. - -THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the -events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall -per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving -one here: - - customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant - product `routes_products.scoped_pool` behind the `product_data` grant - ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall - (404 unknown / 403 not yours — a user table has no module grant). - ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every - ROW to derive a pid set this module discards, and RAISES 409 on a - read-through grid past one window, which is why the record drawer painted - an error page on `ut_odoo_gl_lines`. - -⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a -section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning -another session's gate mid-wave — the WALL is the query parameter, not the word. - -⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every -shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a -400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for -the same reason: a typo served as `customer` answers a question nobody asked). -""" -from fastapi import APIRouter, Body, Depends, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table -#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. -_CUSTOMER_SCOPES = ("", "customer", "cohort") - - -def _scope_or_400(raw): - scope = str(raw or "customer").strip().lower() - if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): - return "customer" if scope in _CUSTOMER_SCOPES else scope - raise err(400, "bad_scope", - "scope must be customer, cohort, product or a ut_ database — refusing to guess") - - -def _pool_or_refuse(session: Session, scope: str): - """The pids this session may attach comments to ON THIS DATABASE — grant wall included. - - Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, - so a caller who may not open the surface never learns anything about the row they asked about. - - ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past - one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns - `frozenset()` for both — see the branch below. - ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that - do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` - that REPLACES this function with its own lambda. A gate's test double is a caller - ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a - bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no - failing name. Change the shape here and that double changes with it. - """ - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - session.require(PRODUCT_MODULE) - pids, _team, _rows, _fields = scoped_pool(session) - # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes - # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised - # a `TypeError` — a 500 on every product comment — while both other branches worked. A - # return shape is a contract even when the function is private. - return (pids, False) - if scope.startswith("ut_"): - # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS - # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the - # rows routes do. - # - # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. - # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on - # a read-through grid larger than one window it RAISES `409 window_required`. So opening the - # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel - # that renders comments about ONE row it already has. `scoped_pids` answers the identical - # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes - # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` - # used to become openable on those two grids. - from routes_tables import scoped_pids - - limits = [] - pids, _fields, _defn = scoped_pids(session, scope, limits=limits) - # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is - # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a - # database with no rows and for a read-through grid too big to enumerate — it distinguishes - # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move - # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which - # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). - # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database - # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the - # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside - # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. - return (pids, bool(limits)) - from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids - - session.require(CUSTOMER_MODULE) - return (frozenset(allowed_pids(session)), False) - - -def _in_book(pid, session, scope): - pids, unbounded = _pool_or_refuse(session, scope) - if unbounded: - # The table wall passed and the row set is larger than this process will enumerate. Said - # out loud rather than silently admitting: R6's second sentence is that a limit which - # cannot be removed gets REPORTED, and this is the one place the report has no envelope to - # ride in. - print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " - f"admitting on the table wall alone, per D-72") - return - if pid not in pids: - # 403, not 404: the record may exist, but this session may not inspect it. - raise err(403, "out_of_scope", "that record is not in your book") - - -def _unavailable(): - return err( - 503, - "store_unavailable", - "record comments are temporarily unavailable — no change was saved", - ) - - -# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD -# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the -# wall was always the query param). The old path stays as an ALIAS because the shipped client -# still calls it; verify_api pins the canonical path AND that the alias answers, so removing -# the alias later is a decision, never an accident. -@router.get("/records/{pid}/comments") -@router.get("/customers/{pid}/comments") -def comments(pid: int, scope: str = Query(default="customer"), - session: Session = Depends(require_session)): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - rows = record_comments.list_comments(session.runtime, pid, scope=scope) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comments": rows} - - -@router.post("/records/{pid}/comments", status_code=201) -@router.post("/customers/{pid}/comments", status_code=201) -def create_comment( - pid: int, - body: dict = Body(default=None), - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - comment = record_comments.add_comment( - session.runtime, - pid, - (body or {}).get("body"), - session.uname, - session.user.get("name") or session.uname, - scope=scope, - ) - except ValueError as exc: - raise err(400, "bad_comment", str(exc)) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comment": comment} - - -@router.delete("/records/{pid}/comments/{comment_id}") -@router.delete("/customers/{pid}/comments/{comment_id}") -def remove_comment( - pid: int, - comment_id: str, - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - deleted = record_comments.delete_comment( - session.runtime, - pid, - comment_id, - session.uname, - admin=session.admin, - scope=scope, - ) - except record_comments.CommentForbidden: - raise err(403, "comment_forbidden", "only the author may delete this comment") - except record_comments.CommentsUnavailable: - raise _unavailable() - if not deleted: - raise err(404, "comment_not_found", "that comment no longer exists") - return {"ok": True, "id": comment_id} +"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. + +⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a +customer-shaped wall bolted to the module import line: `_in_book` asked +`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and +typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the +panel answered "that customer is not in your book" — the owner's report. The dangerous half is +the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment +is filed against somebody's customer where the whole team can read it. + +THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the +events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall +per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving +one here: + + customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant + product `routes_products.scoped_pool` behind the `product_data` grant + ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall + (404 unknown / 403 not yours — a user table has no module grant). + ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every + ROW to derive a pid set this module discards, and RAISES 409 on a + read-through grid past one window, which is why the record drawer painted + an error page on `ut_odoo_gl_lines`. + +⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a +section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning +another session's gate mid-wave — the WALL is the query parameter, not the word. + +⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every +shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a +400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for +the same reason: a typo served as `customer` answers a question nobody asked). +""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table +#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. +_CUSTOMER_SCOPES = ("", "customer", "cohort") + + +def _scope_or_400(raw): + scope = str(raw or "customer").strip().lower() + if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): + return "customer" if scope in _CUSTOMER_SCOPES else scope + raise err(400, "bad_scope", + "scope must be customer, cohort, product or a ut_ database — refusing to guess") + + +def _pool_or_refuse(session: Session, scope: str): + """The pids this session may attach comments to ON THIS DATABASE — grant wall included. + + Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, + so a caller who may not open the surface never learns anything about the row they asked about. + + ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past + one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns + `frozenset()` for both — see the branch below. + ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that + do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` + that REPLACES this function with its own lambda. A gate's test double is a caller + ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a + bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no + failing name. Change the shape here and that double changes with it. + """ + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + session.require(PRODUCT_MODULE) + pids, _team, _rows, _fields = scoped_pool(session) + # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes + # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised + # a `TypeError` — a 500 on every product comment — while both other branches worked. A + # return shape is a contract even when the function is private. + return (pids, False) + if scope.startswith("ut_"): + # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS + # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the + # rows routes do. + # + # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. + # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on + # a read-through grid larger than one window it RAISES `409 window_required`. So opening the + # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel + # that renders comments about ONE row it already has. `scoped_pids` answers the identical + # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes + # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` + # used to become openable on those two grids. + from routes_tables import scoped_pids + + limits = [] + pids, _fields, _defn = scoped_pids(session, scope, limits=limits) + # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is + # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a + # database with no rows and for a read-through grid too big to enumerate — it distinguishes + # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move + # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which + # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). + # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database + # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the + # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside + # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. + return (pids, bool(limits)) + from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids + + session.require(CUSTOMER_MODULE) + return (frozenset(allowed_pids(session)), False) + + +def _in_book(pid, session, scope): + pids, unbounded = _pool_or_refuse(session, scope) + if unbounded: + # The table wall passed and the row set is larger than this process will enumerate. Said + # out loud rather than silently admitting: R6's second sentence is that a limit which + # cannot be removed gets REPORTED, and this is the one place the report has no envelope to + # ride in. + print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " + f"admitting on the table wall alone, per D-72") + return + if pid not in pids: + # 403, not 404: the record may exist, but this session may not inspect it. + raise err(403, "out_of_scope", "that record is not in your book") + + +def _unavailable(): + return err( + 503, + "store_unavailable", + "record comments are temporarily unavailable — no change was saved", + ) + + +# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD +# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the +# wall was always the query param). The old path stays as an ALIAS because the shipped client +# still calls it; verify_api pins the canonical path AND that the alias answers, so removing +# the alias later is a decision, never an accident. +@router.get("/records/{pid}/comments") +@router.get("/customers/{pid}/comments") +def comments(pid: int, scope: str = Query(default="customer"), + session: Session = Depends(require_session)): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + rows = record_comments.list_comments(session.runtime, pid, scope=scope) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comments": rows} + + +@router.post("/records/{pid}/comments", status_code=201) +@router.post("/customers/{pid}/comments", status_code=201) +def create_comment( + pid: int, + body: dict = Body(default=None), + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + comment = record_comments.add_comment( + session.runtime, + pid, + (body or {}).get("body"), + session.uname, + session.user.get("name") or session.uname, + scope=scope, + ) + except ValueError as exc: + raise err(400, "bad_comment", str(exc)) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comment": comment} + + +@router.delete("/records/{pid}/comments/{comment_id}") +@router.delete("/customers/{pid}/comments/{comment_id}") +def remove_comment( + pid: int, + comment_id: str, + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + deleted = record_comments.delete_comment( + session.runtime, + pid, + comment_id, + session.uname, + admin=session.admin, + scope=scope, + ) + except record_comments.CommentForbidden: + raise err(403, "comment_forbidden", "only the author may delete this comment") + except record_comments.CommentsUnavailable: + raise _unavailable() + if not deleted: + raise err(404, "comment_not_found", "that comment no longer exists") + return {"ok": True, "id": comment_id} diff --git a/api/routes_script_views.py b/api/routes_script_views.py index 8db6fc5871a3e0b8f16bd0f077af8552c663cab6..a4feec967331ab1c46fa8a1d6a789eda61b6f63a 100644 --- a/api/routes_script_views.py +++ b/api/routes_script_views.py @@ -1,439 +1,439 @@ -"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). - -Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user -can build whatever they want through the Agent chat interface. be able to create any dashboard -they want. User should have the ability to see the code AND the dashboard output of course… Limit -the code script View per database… Any agent can add into more AI script, so we can see different -versions or different things the AI code for us."* - - GET /api/v1/script-views?database=K the views bound to ONE database - POST /api/v1/script-views create one {database, name?, source} - GET /api/v1/script-views/{id} one view, its source and its history - PUT /api/v1/script-views/{id} a NEW VERSION of the source - DELETE /api/v1/script-views/{id} drop it - POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} - POST /api/v1/script-views/{id}/revert go back to an earlier version {version} - -⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may -read ONLY the database it lives in, and a script that names another database is REFUSED with a -message naming both. *Not capped*: there is **no limit on how many script views a database may -carry**, because that is how an agent offers three attempts and the owner picks one. So nothing -below counts views. What IS bounded is what makes them big — one source is capped, and one view's -edit history is capped and REPORTS what it dropped. - -⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed -`session.user`, so a script written by an administrator and opened by a scoped analyst reads the -ANALYST's rows. The author decides what the code does; the reader decides what it can see. -⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script -that exfiltrates anything, because the only thing a script can return is a render spec drawn on -the screen of the person who ran it. There is no network, no file and no second reader. - -⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a -coroutine that would block the event loop for every other request in the container. FastAPI runs a -plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. -""" -import threading -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. -VIEWS_KEY = "script_views" - -MAX_NAME = 80 -MAX_SOURCE_BYTES = 128 * 1024 - -#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far -#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader -#: can see the history is partial instead of concluding the view was only ever saved twice. -MAX_HISTORY = 40 - -#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal -#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, -#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what -#: gives up. A 429 that says so is honest; an unbounded fork is not. -MAX_CONCURRENT_RUNS = 4 -_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this database has no script views", never to a 500 on the view rail.""" - try: - found = rt.get(VIEWS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _database_ok(session, database): - """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. - - ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True - for an ADMIN on any key at all, including one no database answers to. So a create validated - with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing - can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it - RAISES `UnknownTable`, which is exactly the question being asked. - """ - import core.perm_scope as perm_scope - try: - perm_scope.scoped_fields(session.user, database, st=session.runtime) - except perm_scope.UnknownTable: - raise err(404, "no_database", f"there is no database '{database}' in this workspace") - except perm_scope.Denied: - raise err(403, "forbidden", f"your account may not read '{database}'") - except perm_scope.Unresolvable as exc: - # The database is real and cannot be served under this call's constraints. Standing rule - # 1's second sentence: report the cause and the recommendation, never a bare refusal. - raise err(409, "unresolvable", str(exc)) from None - - -def _clean_source(raw, *, allow_empty=False): - """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. - - ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. - Picking "Custom View" in the mode picker mints the view immediately, before a line of code - exists, so that `mode === 'script'` always implies a real id and no reader has to carry a - "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). - A view that has been created and not yet written is therefore a REAL, legible state: the editor - shows its starter placeholder and the Run control is right there. - A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a - working script by saving nothing over it is not an edit anybody means to make. So the guard - stays exactly where it was on that door, and a person who wants the view gone deletes it. - - ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns - None, so "an empty script is storable" looked true and was written into a mailbox answer another - lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox - knows nothing about. Two validators for one question, disagreeing - [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no - unit of this feature was asserting. - """ - source = str(raw or "") - if not source.strip() and not allow_empty: - raise err(400, "no_source", "a script view needs some code") - if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - raise err(413, "source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") - return source - - -def _row(rec, *, source=False): - """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" - out = {"id": rec.get("id") or "", "database": rec.get("database") or "", - "name": rec.get("name") or "", "author": rec.get("author") or "", - "version": int(rec.get("version") or 1), - "created": rec.get("created") or "", "updated": rec.get("updated") or "", - "versions": len(rec.get("history") or []) + 1, - "trimmed": int(rec.get("trimmed") or 0), - # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the - # row (not only the history) because the editor's header is where a person reads "what - # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back - # legible as an event rather than as a coincidence of matching code. - "restoredFrom": rec.get("restoredFrom")} - if source: - out["source"] = rec.get("source") or "" - out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", - "created": h.get("created") or "", - "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} - for h in reversed(rec.get("history") or []) if isinstance(h, dict)] - return out - - -def _limits(): - return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, - "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, - # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. - "maxViewsPerDatabase": None} - - -def _put(session, view_id, mutate): - """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" - def _set(cur): - cur = dict(cur or {}) - nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) - if nxt is None: - cur.pop(view_id, None) - else: - cur[view_id] = nxt - return cur - - session.runtime.update(VIEWS_KEY, _set, flush="sync") - - -def _mine_or_admin(session, rec): - """Who may EDIT or DELETE a view: its author, or an administrator. - - ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database - may run any view on it, under their OWN scope. That is the whole of "so we can see different - versions or different things the AI code for us": a colleague's attempt is worth nothing if - only its author can open it. - """ - if session.admin or str(rec.get("author") or "") == session.uname: - return - raise err(403, "forbidden", "only the author or an administrator can change this script view") - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/script-views") -def list_script_views(database: str = "", session: Session = Depends(require_session)): - """Every script view bound to ONE database, newest first. `database` is required.""" - key = str(database or "").strip() - if not key: - raise err(400, "no_database", "name the database whose script views you want") - _database_ok(session, key) - rows = [_row(rec) for rec in _all(session.runtime).values() - if isinstance(rec, dict) and str(rec.get("database") or "") == key] - rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) - return {"database": key, "views": rows, "limits": _limits()} - - -@router.post("/script-views") -def create_script_view(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. - - A script view that cannot be run is a broken feature the reader discovers by pressing a button, - and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, - so the refusal arrives while the author still has the code in front of them. - """ - import secrets # noqa: PLC0415 - import core.script_sandbox as sandbox # noqa: PLC0415 - - body = body if isinstance(body, dict) else {} - database = str(body.get("database") or "").strip() - if not database: - raise err(400, "no_database", "a script view is bound to one database") - _database_ok(session, database) - # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the - # moment a person chooses the mode, so the empty source is the normal first state, not a slip. - source = _clean_source(body.get("source"), allow_empty=True) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - view_id = "sv_" + secrets.token_urlsafe(9) - rec = {"id": view_id, "database": database, - "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], - "source": source, "author": session.uname, "version": 1, - "created": _now(), "updated": _now(), "history": [], "trimmed": 0} - _put(session, view_id, lambda _prior: rec) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell the author their - # script was saved when it was not. - raise err(503, "store_unavailable", "the script view was NOT created") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.get("/script-views/{view_id}") -def get_script_view(view_id: str, session: Session = Depends(require_session)): - """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _database_ok(session, str(rec.get("database") or "")) - return {"view": _row(rec, source=True), "limits": _limits()} - - -@router.put("/script-views/{view_id}") -def update_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - source = _clean_source(body.get("source")) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - def _mutate(prior): - prior = dict(prior or rec) - history = list(prior.get("history") or []) - history.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", "created": prior.get("updated") or ""}) - dropped = max(0, len(history) - MAX_HISTORY) - prior["history"] = history[dropped:] if dropped else history - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - if body.get("name"): - prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the new version was NOT saved") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.post("/script-views/{view_id}/revert") -def revert_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Put one view back to an earlier version. D-338 / W37-T46. - - ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole - point of the ticket rather than an implementation detail. The history is what lets a person - trust an agent with their code: if reverting destroyed the versions it stepped over, one - mistaken revert would cost exactly what the history existed to protect, and there would be no - way back from the way back. So this appends, and `restoredFrom` records where the text came - from — the same posture the agent-harness roll-back already takes, so a reader who has seen one - is not surprised by the other. - - ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history - entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old - version's text to re-PUT it. Handing the source out just so the client could send it straight - back would widen a payload for a round trip that does not need to exist, and would make the - revert non-atomic: two calls, and a failure between them leaves the person looking at code that - is not what is stored. - ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was - first saved, and a source that a later, stricter rule would now reject is exactly the source a - person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is - about what the code would DO and that must never be bypassed by a route. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - try: - want = int(body.get("version")) - except (TypeError, ValueError): - raise err(400, "no_version", "say which version to go back to") from None - - history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] - match = next((h for h in history if int(h.get("version") or 0) == want), None) - if match is None: - # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never - # written; the other was TRIMMED off the 40-deep history and the record even counts how - # many. Saying "that version is gone" for the first would be a lie a person could act on. - trimmed = int(rec.get("trimmed") or 0) - if trimmed and want <= trimmed: - raise err(410, "version_trimmed", - f"version {want} is older than this view's history keeps " - f"({trimmed} earlier versions have been dropped)") - raise err(404, "no_version", f"this view has no version {want}") - - source = str(match.get("source") or "") - refusal = sandbox.check_source(source) - if refusal is not None: - # An older version that today's sandbox refuses. The person is told which version and why, - # rather than being handed a 400 about code they did not just type. - raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") - - def _mutate(prior): - prior = dict(prior or rec) - history_now = list(prior.get("history") or []) - history_now.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", - "created": prior.get("updated") or ""}) - dropped = max(0, len(history_now) - MAX_HISTORY) - prior["history"] = history_now[dropped:] if dropped else history_now - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that - # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical - # code and no way to tell a revert from a coincidence. - prior["restoredFrom"] = want - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the roll-back was NOT saved") - return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} - - -@router.delete("/script-views/{view_id}") -def delete_script_view(view_id: str, session: Session = Depends(require_session)): - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - _put(session, view_id, lambda _prior: None) - return {"deleted": view_id} - - -@router.post("/script-views/{view_id}/run") -def run_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. - - ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser - executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key - before this function ever sees it, so a renderer cannot be talked into running something by a - script that was itself perfectly well behaved. - - ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess - wait on the event loop is a ten-second outage for the whole container. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - database = str(rec.get("database") or "") - # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. - # It is checked exactly as a stored one is, because "unsaved" is not a permission. - draft = (body or {}).get("source") if isinstance(body, dict) else None - source = _clean_source(draft) if draft else str(rec.get("source") or "") - # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and - # asking twice builds a registry topic's pool twice. The three refusals are translated below - # instead, which is the same wall reached through the same door. - - if not _RUN_SLOTS.acquire(blocking=False): - raise err(429, "busy", - f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " - f"Try again in a moment") - try: - out = sandbox.run_view(session.user, database, source, st=session.runtime) - finally: - _RUN_SLOTS.release() - - # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" - # answered 200 would be a permission decision the client has to go looking for, and every - # other door in this app answers 403 for it. Everything BELOW this line is a well-formed - # request whose ANSWER is that the script did not produce a view — that is a 200 with - # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. - if out.get("code") == "unknown_table": - raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") - if out.get("code") == "denied": - raise err(403, "forbidden", out.get("message") or "your account may not read that database") - if out.get("code") == "unresolvable": - refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") - refusal.detail["error"]["limit"] = out.get("limit") or {} - raise refusal - - answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), - "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), - "ms": int(out.get("ms") or 0), "code": out.get("code") or "", - # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits - # were applied; on a Windows host the memory and CPU ones were not, and a screen - # that claims an enforcement which did not happen is the failure the rule is about. - "caps": out.get("caps") or {}} - if not answer["ok"]: - answer["error"] = out.get("message") or "the script view did not produce a view" - return answer +"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). + +Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user +can build whatever they want through the Agent chat interface. be able to create any dashboard +they want. User should have the ability to see the code AND the dashboard output of course… Limit +the code script View per database… Any agent can add into more AI script, so we can see different +versions or different things the AI code for us."* + + GET /api/v1/script-views?database=K the views bound to ONE database + POST /api/v1/script-views create one {database, name?, source} + GET /api/v1/script-views/{id} one view, its source and its history + PUT /api/v1/script-views/{id} a NEW VERSION of the source + DELETE /api/v1/script-views/{id} drop it + POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} + POST /api/v1/script-views/{id}/revert go back to an earlier version {version} + +⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may +read ONLY the database it lives in, and a script that names another database is REFUSED with a +message naming both. *Not capped*: there is **no limit on how many script views a database may +carry**, because that is how an agent offers three attempts and the owner picks one. So nothing +below counts views. What IS bounded is what makes them big — one source is capped, and one view's +edit history is capped and REPORTS what it dropped. + +⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed +`session.user`, so a script written by an administrator and opened by a scoped analyst reads the +ANALYST's rows. The author decides what the code does; the reader decides what it can see. +⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script +that exfiltrates anything, because the only thing a script can return is a render spec drawn on +the screen of the person who ran it. There is no network, no file and no second reader. + +⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a +coroutine that would block the event loop for every other request in the container. FastAPI runs a +plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. +""" +import threading +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. +VIEWS_KEY = "script_views" + +MAX_NAME = 80 +MAX_SOURCE_BYTES = 128 * 1024 + +#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far +#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader +#: can see the history is partial instead of concluding the view was only ever saved twice. +MAX_HISTORY = 40 + +#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal +#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, +#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what +#: gives up. A 429 that says so is honest; an unbounded fork is not. +MAX_CONCURRENT_RUNS = 4 +_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this database has no script views", never to a 500 on the view rail.""" + try: + found = rt.get(VIEWS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _database_ok(session, database): + """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. + + ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True + for an ADMIN on any key at all, including one no database answers to. So a create validated + with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing + can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it + RAISES `UnknownTable`, which is exactly the question being asked. + """ + import core.perm_scope as perm_scope + try: + perm_scope.scoped_fields(session.user, database, st=session.runtime) + except perm_scope.UnknownTable: + raise err(404, "no_database", f"there is no database '{database}' in this workspace") + except perm_scope.Denied: + raise err(403, "forbidden", f"your account may not read '{database}'") + except perm_scope.Unresolvable as exc: + # The database is real and cannot be served under this call's constraints. Standing rule + # 1's second sentence: report the cause and the recommendation, never a bare refusal. + raise err(409, "unresolvable", str(exc)) from None + + +def _clean_source(raw, *, allow_empty=False): + """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. + + ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. + Picking "Custom View" in the mode picker mints the view immediately, before a line of code + exists, so that `mode === 'script'` always implies a real id and no reader has to carry a + "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). + A view that has been created and not yet written is therefore a REAL, legible state: the editor + shows its starter placeholder and the Run control is right there. + A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a + working script by saving nothing over it is not an edit anybody means to make. So the guard + stays exactly where it was on that door, and a person who wants the view gone deletes it. + + ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns + None, so "an empty script is storable" looked true and was written into a mailbox answer another + lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox + knows nothing about. Two validators for one question, disagreeing + [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no + unit of this feature was asserting. + """ + source = str(raw or "") + if not source.strip() and not allow_empty: + raise err(400, "no_source", "a script view needs some code") + if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + raise err(413, "source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") + return source + + +def _row(rec, *, source=False): + """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" + out = {"id": rec.get("id") or "", "database": rec.get("database") or "", + "name": rec.get("name") or "", "author": rec.get("author") or "", + "version": int(rec.get("version") or 1), + "created": rec.get("created") or "", "updated": rec.get("updated") or "", + "versions": len(rec.get("history") or []) + 1, + "trimmed": int(rec.get("trimmed") or 0), + # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the + # row (not only the history) because the editor's header is where a person reads "what + # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back + # legible as an event rather than as a coincidence of matching code. + "restoredFrom": rec.get("restoredFrom")} + if source: + out["source"] = rec.get("source") or "" + out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", + "created": h.get("created") or "", + "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} + for h in reversed(rec.get("history") or []) if isinstance(h, dict)] + return out + + +def _limits(): + return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, + "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, + # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. + "maxViewsPerDatabase": None} + + +def _put(session, view_id, mutate): + """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" + def _set(cur): + cur = dict(cur or {}) + nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) + if nxt is None: + cur.pop(view_id, None) + else: + cur[view_id] = nxt + return cur + + session.runtime.update(VIEWS_KEY, _set, flush="sync") + + +def _mine_or_admin(session, rec): + """Who may EDIT or DELETE a view: its author, or an administrator. + + ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database + may run any view on it, under their OWN scope. That is the whole of "so we can see different + versions or different things the AI code for us": a colleague's attempt is worth nothing if + only its author can open it. + """ + if session.admin or str(rec.get("author") or "") == session.uname: + return + raise err(403, "forbidden", "only the author or an administrator can change this script view") + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/script-views") +def list_script_views(database: str = "", session: Session = Depends(require_session)): + """Every script view bound to ONE database, newest first. `database` is required.""" + key = str(database or "").strip() + if not key: + raise err(400, "no_database", "name the database whose script views you want") + _database_ok(session, key) + rows = [_row(rec) for rec in _all(session.runtime).values() + if isinstance(rec, dict) and str(rec.get("database") or "") == key] + rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) + return {"database": key, "views": rows, "limits": _limits()} + + +@router.post("/script-views") +def create_script_view(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. + + A script view that cannot be run is a broken feature the reader discovers by pressing a button, + and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, + so the refusal arrives while the author still has the code in front of them. + """ + import secrets # noqa: PLC0415 + import core.script_sandbox as sandbox # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "no_database", "a script view is bound to one database") + _database_ok(session, database) + # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the + # moment a person chooses the mode, so the empty source is the normal first state, not a slip. + source = _clean_source(body.get("source"), allow_empty=True) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + view_id = "sv_" + secrets.token_urlsafe(9) + rec = {"id": view_id, "database": database, + "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], + "source": source, "author": session.uname, "version": 1, + "created": _now(), "updated": _now(), "history": [], "trimmed": 0} + _put(session, view_id, lambda _prior: rec) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell the author their + # script was saved when it was not. + raise err(503, "store_unavailable", "the script view was NOT created") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.get("/script-views/{view_id}") +def get_script_view(view_id: str, session: Session = Depends(require_session)): + """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _database_ok(session, str(rec.get("database") or "")) + return {"view": _row(rec, source=True), "limits": _limits()} + + +@router.put("/script-views/{view_id}") +def update_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + def _mutate(prior): + prior = dict(prior or rec) + history = list(prior.get("history") or []) + history.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", "created": prior.get("updated") or ""}) + dropped = max(0, len(history) - MAX_HISTORY) + prior["history"] = history[dropped:] if dropped else history + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + if body.get("name"): + prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the new version was NOT saved") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.post("/script-views/{view_id}/revert") +def revert_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Put one view back to an earlier version. D-338 / W37-T46. + + ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole + point of the ticket rather than an implementation detail. The history is what lets a person + trust an agent with their code: if reverting destroyed the versions it stepped over, one + mistaken revert would cost exactly what the history existed to protect, and there would be no + way back from the way back. So this appends, and `restoredFrom` records where the text came + from — the same posture the agent-harness roll-back already takes, so a reader who has seen one + is not surprised by the other. + + ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history + entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old + version's text to re-PUT it. Handing the source out just so the client could send it straight + back would widen a payload for a round trip that does not need to exist, and would make the + revert non-atomic: two calls, and a failure between them leaves the person looking at code that + is not what is stored. + ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was + first saved, and a source that a later, stricter rule would now reject is exactly the source a + person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is + about what the code would DO and that must never be bypassed by a route. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + try: + want = int(body.get("version")) + except (TypeError, ValueError): + raise err(400, "no_version", "say which version to go back to") from None + + history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] + match = next((h for h in history if int(h.get("version") or 0) == want), None) + if match is None: + # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never + # written; the other was TRIMMED off the 40-deep history and the record even counts how + # many. Saying "that version is gone" for the first would be a lie a person could act on. + trimmed = int(rec.get("trimmed") or 0) + if trimmed and want <= trimmed: + raise err(410, "version_trimmed", + f"version {want} is older than this view's history keeps " + f"({trimmed} earlier versions have been dropped)") + raise err(404, "no_version", f"this view has no version {want}") + + source = str(match.get("source") or "") + refusal = sandbox.check_source(source) + if refusal is not None: + # An older version that today's sandbox refuses. The person is told which version and why, + # rather than being handed a 400 about code they did not just type. + raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") + + def _mutate(prior): + prior = dict(prior or rec) + history_now = list(prior.get("history") or []) + history_now.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", + "created": prior.get("updated") or ""}) + dropped = max(0, len(history_now) - MAX_HISTORY) + prior["history"] = history_now[dropped:] if dropped else history_now + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that + # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical + # code and no way to tell a revert from a coincidence. + prior["restoredFrom"] = want + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the roll-back was NOT saved") + return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} + + +@router.delete("/script-views/{view_id}") +def delete_script_view(view_id: str, session: Session = Depends(require_session)): + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + _put(session, view_id, lambda _prior: None) + return {"deleted": view_id} + + +@router.post("/script-views/{view_id}/run") +def run_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. + + ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser + executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key + before this function ever sees it, so a renderer cannot be talked into running something by a + script that was itself perfectly well behaved. + + ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess + wait on the event loop is a ten-second outage for the whole container. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + database = str(rec.get("database") or "") + # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. + # It is checked exactly as a stored one is, because "unsaved" is not a permission. + draft = (body or {}).get("source") if isinstance(body, dict) else None + source = _clean_source(draft) if draft else str(rec.get("source") or "") + # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and + # asking twice builds a registry topic's pool twice. The three refusals are translated below + # instead, which is the same wall reached through the same door. + + if not _RUN_SLOTS.acquire(blocking=False): + raise err(429, "busy", + f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " + f"Try again in a moment") + try: + out = sandbox.run_view(session.user, database, source, st=session.runtime) + finally: + _RUN_SLOTS.release() + + # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" + # answered 200 would be a permission decision the client has to go looking for, and every + # other door in this app answers 403 for it. Everything BELOW this line is a well-formed + # request whose ANSWER is that the script did not produce a view — that is a 200 with + # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. + if out.get("code") == "unknown_table": + raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") + if out.get("code") == "denied": + raise err(403, "forbidden", out.get("message") or "your account may not read that database") + if out.get("code") == "unresolvable": + refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") + refusal.detail["error"]["limit"] = out.get("limit") or {} + raise refusal + + answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), + "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), + "ms": int(out.get("ms") or 0), "code": out.get("code") or "", + # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits + # were applied; on a Windows host the memory and CPU ones were not, and a screen + # that claims an enforcement which did not happen is the failure the rule is about. + "caps": out.get("caps") or {}} + if not answer["ok"]: + answer["error"] = out.get("message") or "the script view did not produce a view" + return answer diff --git a/api/routes_shares.py b/api/routes_shares.py index 5d95fbaccad34ee7b76fa887a6ab5f9cd2e8f1f7..1d4d09210f8d7db546a1590018ef9ca02b574063 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1,789 +1,929 @@ -"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). - - GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} - PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) - GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} - -`kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the -view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE -vocabulary in the UI (R10: "the same picker views use"). - -⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `":"`** -(`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen — -so a grant stored under one would admit the grantee to every `notes` column in the tenant at -once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not -obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator -is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None, -`_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item -18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for -anything that is not a view. - -⭐⭐ **A "Can edit" GRANTEE MAY RE-SHARE A VIEW — OWNER RULING R4, BUILT AS W40-T02** (instruction -4: *"Edit View so a member can share a View as well, not just an admin"*). This REVERSES the flat -owner-or-admin sentence that stood here, and the reversal is bounded three ways, all enforced HERE -and not in the client: - - 1. **ONLY `kind='view'`.** `shares.RESHARE_KINDS` is the one spelling of that. `folder`, - `database` and `field` still require the owner or an admin, and the paragraph below is why - the `database` kind in particular was never a candidate: a `ut_*` grant's blast radius is a - whole table, where a view is one saved SELECTION over rows the receiver's own wall governs. - 2. **A RE-SHARE MAY NEVER EXCEED THE RE-SHARER'S OWN ROLE**, and the strict reading ships: an - `edit` grantee may hand out `view` and NEVER `edit`. `shares.max_grantable_role` answers the - ceiling, `put_share` enforces it on the DELTA (a name arriving at `edit`, or one raised to - it) — never on every row of the body, because the `PUT` REPLACES and `ShareDialog.save` - therefore re-sends the whole list with the re-sharer's own `edit` row inside it. Conferring - `edit` stays the owner's alone. - 3. **OWNERSHIP NEVER MOVES.** A caller re-sharing rather than owning passes the EXISTING owner - straight through (`put_share` below, stated rather than incidental). - -⭐⭐ **AND R4 BOUNDED ONLY THE ROLE. TWO MORE BOUNDS ARRIVED WITH THE OWNER'S RULING OF -2026-08-24, BECAUSE THE GAPS WERE MEASURED ON THE SHIPPED BUILD:** - - 4. **A RE-SHARER MAY SHARE ONLY WITH SPECIFIC PEOPLE (D-473).** Owner: *"a re-sharer can only - share to specific people"*. Before this, `[fisch:edit, *:view]` from an `edit` grantee was a - `200` — the role ceiling could not see it, because `*` at `view` never exceeds a `view` - ceiling. `_audience_added` refuses an `EVERYONE` entry this caller is ADDING, and lets one - the OWNER already placed ride along untouched, for the same delta reason as (2). - 5. **A RE-SHARER MAY UN-SHARE ONLY THE PEOPLE THEY SHARED TO (D-474).** Owner: *"a re-sharer - can only unshare the people it shared to"*. Before this, an `edit` grantee PUTting a list - that omitted another grantee got `200` and that person's access was gone; `PUT []` left the - view shared with nobody. Answering this needs a fact the registry did not store, so each - entry now carries `by` — who placed it — stamped by `shares.set_grants(granter=…)` and - consulted by `_unremovable`. ⛔ An entry with NO `by` (every grant predating the change) is - the owner's or an admin's to revoke and nobody else's: fail-closed, because from the door - "unstamped" and "somebody else's" are the same observation. - -⛔ **SO BOTH FEARS THE OLD SENTENCE NAMED ARE NOW ANSWERED, AND THE PARAGRAPH THAT PRICED THE -SECOND ONE IS GONE RATHER THAN SOFTENED.** It read *"'Widen a users-scoped view to everyone' is -NOT closed — an `edit` grantee may add `*` — it is CAPPED at Can view"*, and that was an accurate -account of R4 that the owner overruled the moment it was put in front of them. *"Grant themselves -ownership and lock you out"* stays closed by (3) and `set_grants`' sticky owner. A `view` grantee -still cannot share at all, and still gets `403`. The client greys the editor for -non-administrators and offers "Can edit" to anyone it greys in; that is a courtesy, and these -checks are the wall. - -⚠ **WHAT `GET` NOW DISCLOSES.** The grant record carries `by`, and this route returns the record -verbatim — so anybody who may read a view's grant list also learns who added each person, a -`view` grantee included. Accepted deliberately (the alternative is a second, stripped read path -and therefore a second answer to one question), and stated here because this file states costs -rather than leaving them to be discovered. - -⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means -every account that can already open the surface: `require_session` plus the topic's own gate run -first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field -closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that -could already reach the data ([[aios-permissioning]]). - -⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED -MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` -and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be -DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes -`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded -by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every -column — and an `*` database grant admits every account in the tenant to all of it. -That is a real capability, deliberately kept; what was wrong was a docstring promising a second -wall that does not exist for this kind. Scoping user tables is booked, not done -(`waves/wave32/sharing-audit.md` S-8). - -⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS -registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` -— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here -whose object is invisible under that one is a row in a list that opens a refusal, which is what -made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but -the two vocabularies are still two. -""" -from fastapi import APIRouter, Body, Depends - -import core.shares as shares -import core.users as users -from deps import Session, err, require_session -# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES -# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the -# reader agreeing about one string is the whole difference between an Inbox row that opens the -# shared database and one that is quietly unclickable. -from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC - -router = APIRouter(prefix="/api/v1") - - -def _kind_or_400(raw): - try: - return shares._check_kind(raw) - except ValueError as e: - raise err(400, "bad_kind", str(e)) - - -# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── -# -# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all -# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free -# strings off the URL and the only dependency was `require_session`, so any signed-in account -# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, -# an object with no grant record skipped the check entirely and the caller was stamped OWNER — -# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not -# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. -# -# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own -# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the -# victim next opens the dialog. - -#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share -#: route is not told which topic — so resolving one means asking each. -_BUILTIN_TOPICS = ("customer", "product") - - -def _field_storage_keys(table_key): - """Resolve the client-facing field topic to its durable stores and grant topic.""" - raw = str(table_key or "").strip() - if raw in ("customer_data", "customer_table_workspace"): - return "customer_table_workspace", "customer_table_workspace", "customer_data" - if raw in ("product_data", "product_table_workspace"): - return "product_table_workspace", "product_table_workspace", "product_data" - if raw.startswith("ut_"): - bare = raw[:-len("_table_workspace")] if raw.endswith("_table_workspace") else raw - return f"{bare}_table_workspace", bare, bare - workspace = raw if raw.endswith("_table_workspace") else f"{raw}_table_workspace" - shared = raw[:-len("_table_workspace")] if raw.endswith("_table_workspace") else raw - return workspace, shared, shared - - -def _field_definition(session, table_key, field_key): - """Return the shared/private definition and the keys used by its write paths.""" - workspace_key, shared_key, grant_topic = _field_storage_keys(table_key) - try: - from core import shared_overlay - shared = (shared_overlay.fields(shared_key, st=session.runtime) or {}).get(field_key) - if isinstance(shared, dict): - return shared, True, workspace_key, shared_key, grant_topic - import core.table_store as table_store - private = (table_store.make(workspace_key, st=session.runtime) - .workspace(session.uname).get("fields") or {}).get(field_key) - if isinstance(private, dict): - return private, False, workspace_key, shared_key, grant_topic - except Exception: # noqa: BLE001 - pass - return None, False, workspace_key, shared_key, grant_topic - - -def _field_owner(session, definition, already_shared): - """Resolve the creator for the share claim wall. - - A private field is already namespaced by the caller's own workspace. Older field records - from before the host-side creator stamp therefore remain safely claimable by that workspace - owner, while a shared definition with no creator stays admin-only because its storage is - tenant-wide and cannot identify an owner from residency alone. - """ - owner = str((definition or {}).get("createdBy") or "").strip() - if owner: - return owner - return str(session.uname or "").strip() if not already_shared else "" - - -def _topics(session): - """Every topic whose workspace could hold a view or folder for this tenant. - - ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on - tenant #0) to answer a question about KEYS (D-185). - """ - try: - import core.user_tables as ut - return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) - except Exception: # noqa: BLE001 - return _BUILTIN_TOPICS - - -def _owns_object(session, kind, oid): - """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? - - ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking - each topic's workspace in turn, which is N store reads; making every share call pay that - would put a loop on a route the manage-access dialog opens. The dangerous path is the one - where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs - exactly there, and the common path (a record exists, `may_administer` decides) is untouched. - """ - if session.admin: - return True - if kind == "field": - # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS - # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 / - # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be - # deletable by one person and shareable by another. - # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION: - # a brand-new column has no grant record, so `put_share` falls to this predicate — and - # without it the column's own creator is answered `404 no_object` on the first attempt to - # share the thing they just made. - table_key, field_key = shares.split_field_oid(oid) - if not table_key: - return False - defn, _shared, _workspace, _shared_key, _grant_topic = _field_definition( - session, table_key, field_key) - owner = _field_owner(session, defn, _shared) - return bool(defn) and owner.lower() == str(session.uname).strip().lower() - if kind == "database": - # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already - # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" - # here would be the second definition this wave keeps finding. - try: - import core.user_tables as ut - return bool(ut.may_open(oid, session.uname, is_admin=session.admin, - st=session.runtime)) - except Exception: # noqa: BLE001 - return False - try: - import core.table_store as table_store - except Exception: # noqa: BLE001 - return False - for topic in _topics(session): - try: - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - except Exception: # noqa: BLE001 - continue - if hit: - # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the - # person whose personal stratum holds it — anybody else reaching this line is - # exactly the case S-1 describes. - return str(hit[0]) == str(session.uname) - return False - - -def _can_see_object(session, kind, oid): - """May this caller READ an object's grant list — i.e. can they reach the object at all? - - ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A - REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the - ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches - PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with - `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit - in the grid** — answered `404` when bob opened its manage-access dialog. Measured before - fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said - `404 no_object`. - ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", - and the wall consulted the grant registry (system A) plus stratum ownership, never the view's - `permissions` (system B) — which is the one that actually decides who may OPEN it. - ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not - gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — - 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a - legitimate collaborator their view does not exist. - - ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its - owner — that is S-1, and widening this predicate onto `put_share` would re-open it. - """ - if _owns_object(session, kind, oid): - return True - if kind != "view": - # A folder carries no per-object visibility flag of its own, and a database's `may_open` - # (inside `_owns_object`) already admits grantees. Nothing wider to ask. - # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here: - # `get_share` tests `role is None` first and a grant answers a role, so the only caller - # left is an account with no relationship to the column at all. Widening it would let any - # signed-in session enumerate who holds which column on a database they cannot open. - return False - try: - import core.table_store as table_store - for topic in _topics(session): - hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) - if hit: - return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, - session.uname, is_admin=session.admin)) - except Exception: # noqa: BLE001 - return False - return False - - -def _entries_or_400(session, entries): - """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. - - ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: - a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE - of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, - so a typo'd name is stored, reported as a successful save, and never reaches anybody. The - sharer believes the person has access. That is item 18's plain reading. - ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker - (`_people`). One route, two populations, and the write door was the permissive one. - ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for - "every account that can already open the surface". - """ - known = {p["username"].strip().lower() for p in _people(session.tenant)} - unknown = [] - for e in entries or (): - if not isinstance(e, dict): - continue - user = str(e.get("user") or "").strip().lower() - if user and user != shares.EVERYONE and user not in known: - unknown.append(user) - if unknown: - raise err(400, "unknown_people", - "no active account in this workspace is named " - + ", ".join(sorted(set(unknown))) - + ". Nothing was shared. Pick people from the list rather than typing a name.") - - -def _named(who): - """The people in a refusal, in the words the person reading it uses. - - ONE spelling, used by all three re-share refusals below. `*` is never printed raw: the store's - wildcard is a single character, and a 403 reading *"set * to Can view"* names nothing a person - can find in the dialog they are looking at. - """ - return ", ".join("everyone in this workspace" if w == shares.EVERYONE else w - for w in sorted(who)) - - -def _audience_added(entries, held): - """⭐⭐ D-473 — the `EVERYONE` grant this caller is ADDING, or the empty set. - - OWNER RULING 2026-08-24, verbatim: *"a re-sharer can only share to specific people"*. R4's - ceiling bounded the ROLE a re-sharer may hand out and said nothing about the AUDIENCE, and the - gap was measured rather than argued: an `edit` grantee PUTting `[fisch:edit, *:view]` was - answered `200`, widening a two-person view to the whole tenant at `Can view`. The role check - could not catch it, because `*` at `view` never exceeds a `view` ceiling. - - ⛔ IT IS THE ADDITION THAT IS REFUSED, NOT THE PRESENCE, and that is the same shape as the - role check one arm above, for the same reason: the `PUT` REPLACES, so `ShareDialog.save` - re-sends the WHOLE list every time. An `*` the OWNER placed rides along in every payload the - re-sharer is able to produce, and refusing on presence would `403` every save on a view the - owner had already opened to everyone — a re-sharer locked out of a list they may legitimately - edit, with a message about a row they did not touch. - - ⛔ ROLE IS NOT CONSULTED HERE, DELIBERATELY. This answers "may this caller widen the - AUDIENCE", and `*` held at `view` and resubmitted at `edit` is a ROLE escalation that the - check above already refuses, by name. Two questions, two predicates - ([[one-evaluator-per-question]]) — and separable is also what lets a gate disarm one of them - in memory and prove the other still fires. - - ⚠ `_clean_entries`, NEVER THE RAW BODY: an entry with a junk role is dropped by the writer, - so reading the raw list would refuse a widening that was never going to be stored. - """ - if shares.EVERYONE in (held or {}): - return set() - return {e["user"] for e in shares._clean_entries(entries) if e["user"] == shares.EVERYONE} - - -def _unremovable(held_rows, entries, uname): - """⭐⭐ D-474 — the people this caller is dropping from the list but may NOT revoke. - - OWNER RULING 2026-08-24, verbatim: *"a re-sharer can only unshare the people it shared to"*. - R4 bounded what a re-sharer may HAND OUT and left what they may TAKE AWAY unbounded, and both - halves of that were measured: an `edit` grantee PUTting a list that omits another grantee was - answered `200` and that person's access was gone; `PUT []` left the view shared with nobody. - - ⛔⛔ AN ENTRY WITH NO `by` IS NOT REMOVABLE BY A RE-SHARER — ONLY BY THE OWNER OR AN ADMIN, - AND THAT IS THE FAIL-CLOSED DIRECTION RATHER THAN AN OVERSIGHT. Every grant written before - provenance existed carries no stamp, so "no `by`" and "granted by somebody else" are - indistinguishable from here. Reading absence as *"nobody claims it, so anyone may take it"* - would hand every re-sharer the power to revoke the entire pre-existing grant set on day one - of this change, which is the exact capability the ruling withholds. A re-sharer must not be - able to revoke a grant they cannot PROVE they made [[aios-permissioning]]. - - ⚠ SO A RE-SHARER CANNOT REMOVE THEMSELVES EITHER, AND THAT IS STATED BECAUSE IT LOOKS LIKE A - BUG. Their own row was placed by the owner, so it carries the owner's `by` and lands in this - set. Read literally, the ruling says a re-sharer unshares only who THEY shared to, and their - own grant is not one of those. Leaving the view is the owner's to do, like every other - revocation of an owner-placed grant. ⛔ Do not carve an exception here without a ruling: the - carve-out is indistinguishable from "a re-sharer may revoke any row whose `by` names the - owner", which is the wall itself. - - ⚠ `_clean_entries`, NEVER THE RAW BODY, AND THIS IS THE HOLE THAT SHAPE CLOSES. A role the - writer rejects is a row that will NOT be stored, so `{user: victim, role: "nonsense"}` looks - present in the raw payload and is a silent REVOCATION once written. Asking the same - normaliser the store uses is what makes "submitted" mean the same thing at both ends. - """ - me = str(uname or "").strip().lower() - submitted = {e["user"] for e in shares._clean_entries(entries)} - stuck = set() - for user, row in (held_rows or {}).items(): - if user in submitted: - continue - stamp = str((row or {}).get("by") or "").strip().lower() - if not me or not stamp or stamp != me: - stuck.add(user) - return stuck - - -@router.get("/share/mine") -def my_shares(session: Session = Depends(require_session)): - """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). - - Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves - in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL - that is not malformed at all. - """ - return shares.shared_with(session.uname, st=session.runtime) - - -@router.get("/share/{kind}/{oid}") -def get_share(kind: str, oid: str, session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - if kind == "field": - table_key, field_key = shares.split_field_oid(oid) - if table_key and field_key: - oid = shares.field_oid(_field_storage_keys(table_key)[2], field_key) - rec = shares.grants(kind, oid, st=session.runtime) - role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) - may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime) - # W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant - # registry has no owner to return from `may_administer`, even though the same caller may safely - # claim their own object through the PUT path below. Reflect that exact claim predicate here: - # a Member who owns an unshared View receives the people picker; a collaborator still does not. - if not rec["owner"] and not may_admin and _owns_object(session, kind, oid): - may_admin = True - # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: - # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — - # to any signed-in session, about objects it cannot open. Now a caller with no role on an - # object must prove they can reach it, and gets a 404 otherwise: the same answer a - # non-existent id gives, so the route cannot be used to probe which ids are real. - # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a - # caller who has no relationship with the object at all. - if role is None and not _can_see_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - return { - **rec, - "role": role, - "mayAdminister": may_admin, - # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry - # them. `assignable_people` serves bare display names because `user`-kind CELLS store - # display names — that list's shape cannot change without migrating cell values — so - # this route serves objects of its own. Existing grants that were written as lowercased - # display names are normalised by the wave-21 cleanup script. - # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller - # who may open the editor. A read-only grantee gets the grant list (their fair question is - # "who else has this?") and not a directory of every account in the workspace. - # ⭐⭐ W40-T02 / R4 — AND THAT RULE IS WHY THIS LINE NEEDED NO EDIT. `may_administer` now - # answers True for an `edit` grantee on a VIEW, which MOVES that account into "may open - # the editor" — so the picker they need arrives by the roster riding on the same flag it - # always did. Gating it on anything else (owner, `role == 'owner'`, a fresh predicate) - # would be a second answer to a question this file already answers once, and would leave - # the new grantee with an editor and no people to put in it. A `view` grantee is still - # `may_admin=False` here and still gets `[]`. - "people": _people(session.tenant) if may_admin else [], - } - - -def _people(tenant): - """[{username, name}] for this tenant — same population as `assignable_people`, with the - BINDING identity alongside the display one.""" - try: - reg = users.registry() or {} - except Exception: - return [] - want = str(tenant or '').strip().lower() - out = [] - for uname, u in reg.items(): - if not isinstance(u, dict) or u.get('active') is False: - continue - if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: - continue - out.append({"username": str(uname), "name": str(u.get('name') or uname)}) - return sorted(out, key=lambda p: p["name"].lower()) - - -@router.put("/share/{kind}/{oid}") -def put_share(kind: str, oid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - if kind == "field": - table_key, field_key = shares.split_field_oid(oid) - if table_key and field_key: - oid = shares.field_oid(_field_storage_keys(table_key)[2], field_key) - body = body or {} - rec = shares.grants(kind, oid, st=session.runtime) - # `claiming` is the "no owner yet, and this caller may become one" branch, hoisted to a name - # because TWO decisions below need it: the ceiling (a claimant is about to be the owner, so - # their ceiling is an owner's) and the owner written back (D3 — see `set_grants` at the end). - claiming = False - # An object with NO grant record yet has no owner — the first person to share it claims it. - # That is safe because reaching this route at all means passing the surface's own wall, and - # the alternative (refusing until somebody seeds an owner) would make a brand-new folder - # unshareable by the person who just made it. - if rec["owner"]: - if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime): - # ⭐ R4 / W40-T02 — `may_administer` now also admits an `edit` grantee on a VIEW, so - # the population refused here is narrower than the code word `not_owner` suggests: a - # `view` grantee, or an account with an `edit` role on a kind outside - # `shares.RESHARE_KINDS`. The code string is kept because clients match on it. - raise err(403, "not_owner", - "only the owner of this item (or an administrator) can change who it is " - "shared with") - # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record - # is still claimed by the first person to share it — that rule is right, and refusing until - # somebody seeds an owner would make a brand-new folder unshareable by the person who just - # made it. What was missing is the half the old comment ASSERTED and the code never did: the - # claimant has to be able to reach the object. Without this, any signed-in account could - # stamp itself owner of an id it had never seen and lock the real creator out for good. - elif not _owns_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - else: - claiming = True - entries = body.get("entries") - if not isinstance(entries, list): - raise err(400, "bad_entries", - "entries must be a list of {user, role}. Send [] to un-share, which is how " - "revoking is expressed") - _entries_or_400(session, entries) - # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit` - # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer - # holds") needs a check of its own: that caller may hand out `view`, and conferring `edit` - # stays the owner's or an administrator's. - # - # ⛔ AFTER THE ADMISSION, NEVER BEFORE, AND THAT ORDER IS A SECURITY PROPERTY. A caller with - # no role at all must keep receiving `404 no_object` (audit S-1/S-3: a stranger learns - # nothing, so this route cannot be used to probe which ids are real). A ceiling raised first - # would answer that caller `403` and turn the one route hardened against id-probing back into - # an oracle that confirms an id exists. It also runs before the `field` promotion below, so a - # refusal cannot leave a column promoted with no grant written. - # - # ⛔ AND IT IS THE DELTA, NOT EVERY ROW OF THE BODY — read off the shipped client, not - # assumed. `ShareDialog.save` PUTs the WHOLE list every time ("a body assembled from a delta - # would revoke everyone it failed to mention"), so the re-sharer's OWN `{user, role: "edit"}` - # row rides in every payload they are able to produce. Refusing per-entry would `403` the - # exact re-share this ticket exists to enable, and the only body that would pass is one that - # revokes the re-sharer. So what is refused is edit access this caller is CREATING: a name - # arriving at `edit`, or an existing `view` grantee raised to it. A row that already stood at - # `edit` was the OWNER's decision, and is not this caller's to be refused for. - # - # ⚠ A CLAIMANT IS AN OWNER. The branch above admits a Member who owns an object that has no - # grant record yet, and `set_grants` is about to stamp them owner — asking the registry for - # their role here would answer `None` (no record exists to hold one) and refuse the first - # `edit` grant on every newly created view. Same predicate as the door, one line apart. - ceiling = "edit" if claiming else shares.max_grantable_role( - kind, oid, session.uname, is_admin=session.admin, st=session.runtime) - # - # ⭐⭐ OWNER RULING 2026-08-24 (D-473 + D-474) — AND THE CEILING IS NOW ONE OF THREE WALLS IN - # THIS BLOCK, NOT THE WALL. R4 bounded the ROLE a re-sharer may hand out and was silent on the - # other two questions a re-share asks, so both gaps shipped and both were measured on the - # build: an `edit` grantee could PUT `[fisch:edit, *:view]` and widen a two-person view to the - # whole tenant (200), and could PUT a list omitting another grantee — or `[]` — and revoke - # people they never granted (200). The owner's answer settles both in one sentence: *"no a - # re-sharer can only share to specific people and a re-sharer can only unshare the people it - # shared to"*. - # - # ⛔ THREE PREDICATES, THREE FUNCTIONS, ONE ORDER: role, then audience, then revocation. They - # are separate because they answer separate questions and because a wall that cannot be - # disarmed ALONE cannot be proven alone — `verify_scopes.section_reshare_bounds` patches each - # one in memory and requires exactly its own leg to go red, which a single fused `if` would - # make impossible ([[a-declared-gate-is-an-unchecked-claim]]). - # ⚠ ROLE AND AUDIENCE COMPOSE, AND THE ORDER DECIDES WHICH REFUSAL A PERSON READS. `*` - # submitted at `edit` while held at `view` is BOTH an escalation and (if unheld) a widening; - # the role check runs first and names the fix that is actually available to this caller - # ("set it to Can view"), which is the more useful of the two sentences. - if ceiling != "edit": - # The prior ROWS, not just their roles: the revocation wall needs each entry's `by`, and - # reading it from a second place would be a second answer to "what does the store hold". - held_rows = {e.get("user"): e for e in (rec["entries"] or ()) - if isinstance(e, dict) and e.get("user")} - held = {u: e.get("role") for u, e in held_rows.items()} - noun = {"view": "view", "folder": "folder", - "database": "database", "field": "column"}.get(kind, "item") - raised = set() - for e in entries: - if not isinstance(e, dict): - continue - who = str(e.get("user") or "").strip().lower() - if who and str(e.get("role") or "").strip().lower() == "edit" \ - and held.get(who) != "edit": - raised.add(who) - if raised: - raise err(403, "grant_exceeds_role", - "you can share this " + noun + " at Can view, which is as far as your own " - "access reaches. Only its owner (or an administrator) can give somebody " - "Can edit, so nothing was saved. Set " + _named(raised) - + " to Can view and save again.") - # ⭐⭐ D-473 — THE AUDIENCE. A re-sharer names PEOPLE; reaching "everyone" is the owner's. - if _audience_added(entries, held): - raise err(403, "grant_exceeds_audience", - "you can share this " + noun + " with specific people, which is as far as " - "your own access reaches. Only its owner (or an administrator) can open it " - "to everyone in this workspace, so nothing was saved. Remove Everyone from " - "the list, add the people you meant by name, and save again.") - # ⭐⭐ D-474 — THE REVOCATION. An omission IS a revocation on a replacing PUT, so this is - # the only place a removal can be refused. ⛔ REFUSED WHOLE: `set_grants` has not run, so - # a payload carrying a legitimate addition ALONGSIDE a forbidden removal saves neither. - # That is deliberate and it is what the message promises ("nothing was saved") — a - # half-applied permission change is worse than a refused one, because the person reading - # the toast has no way to tell which half took. - stuck = _unremovable(held_rows, entries, session.uname) - if stuck: - raise err(403, "revoke_not_yours", - "you can remove the people you shared this " + noun + " with, and this " - "workspace has no record of you sharing it with " + _named(stuck) - + ". Only its owner (or an administrator) can remove them, so nothing was " - "saved. Put them back on the list and save again.") - if kind == "field": - # A field grant is a visibility and edit wall. Promote a private custom - # field exactly once, then keep the requested Share field role as the - # authoritative override for the legacy permissions bag. - from core import field_permissions, shared_overlay - table_key, field_key = shares.split_field_oid(oid) - defn, already_shared, workspace_key, shared_key, grant_topic = _field_definition( - session, table_key, field_key) - if not isinstance(defn, dict): - raise err(404, "no_object", "no such field, or it is not shared with this account") - if not already_shared: - defn = field_permissions.promote_field( - workspace_key, shared_key, grant_topic, - session.uname, defn, st=session.runtime) - stamped = dict(defn) - stamped["shared"] = True - stamped["granted"] = True - shared_overlay.put_field(shared_key, field_key, stamped, st=session.runtime) - oid = shares.field_oid(grant_topic, field_key) - # ⭐⭐ R4 / W40-T02 (D3) — OWNERSHIP NEVER MOVES ON A RE-SHARE, AND IT IS SAID HERE RATHER - # THAN LEFT TO FALL OUT. `set_grants`' owner is sticky, so the old `rec["owner"] or - # session.uname` already happened not to transfer ownership — incidentally, as a property of - # the callee. R4 names ownership transfer as one of the two halves of the old protection that - # SURVIVES the widening, and a rule that survives by accident is one the next edit deletes - # without noticing. So the branch is explicit: a claimant becomes the owner, and everybody - # else — an owner re-saving, an admin, and now an `edit` grantee re-sharing — passes the - # EXISTING owner straight back through. `claiming` is the same flag the admission set, so - # there is no second answer to "is this person taking ownership". - # ⭐⭐ D-474 — `granter` IS THIS SESSION, ON EVERY SAVE INCLUDING THE OWNER'S. Provenance is - # recorded for whoever adds a person, not only for a re-sharer: an owner-placed grant carrying - # NO stamp is indistinguishable from a pre-provenance one, and `_unremovable` would then be - # deciding on the store's AGE rather than on who granted what. `set_grants` stamps only - # entries that are NEW to the record and never re-stamps an existing one, so an owner - # re-saving a list does not quietly take provenance off the re-sharer who built it. - # ⛔⛔ THE THREE REFUSALS ABOVE WERE DECIDED AGAINST `rec`, WHICH WAS READ AT THE TOP OF - # THIS FUNCTION. Handing `expect` to the writer is what makes them true at the moment of the - # write rather than at the moment of the read: a concurrent save lands between the two, and - # a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the - # owner had just added disappeared. See `shares.set_grants`' own note. - try: - out = shares.set_grants(kind, oid, entries, - owner=session.uname if claiming else rec["owner"], - granter=session.uname, - st=session.runtime, - expect=rec["entries"]) - except shares.GrantsChanged as exc: - raise err(409, "grants_changed", str(exc)) - _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) - return out - - -def _notify_new_grantees(session, kind, oid, before, after): - """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. - - Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was - silent: the grant landed in a rail section the receiver had to notice on their own, which is - why "I shared it with you" and "I never saw it" were both true. - - ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read - because an alert is a live question about rows; a share is an EVENT that happened once, and - polling for it would mean re-deriving "was this new?" on every inbox open — the diff below - only exists here, at the moment the set changes. - - ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), - so every save re-sends everyone who was already there. Diffing against `before` is what stops - a rename or a role change from ringing the bell for people whose access did not change. - ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in - the tenant on a single click is a broadcast nobody asked for. The rail still shows it. - ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the - grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. - """ - try: - was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} - fresh = [str(e.get("user")) for e in (after or ()) - if isinstance(e, dict) and e.get("user") not in was - and e.get("user") != shares.EVERYONE] - if not fresh: - return - import core.alerts as alerts - - label, route, view_id = _object_ref(session, kind, oid) - if not route: - # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and - # `notification_view` would have to invent a target. Silence is the honest answer - # here; the rail still shows the grant under "Shared with me". - return - sharer = str(session.user.get("name") or session.uname) - for user in fresh: - # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must - # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes - # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED - # from there rather than typed again — one vocabulary, one owner. - # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY - # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and - # are honestly named by their machine; a SHARE has a real person, and only this call - # site knows who. ⛔ It is passed as its OWN field rather than recovered from the - # `detail` prose below: a sender parsed out of " shared this with you" breaks - # the first time the sentence is reworded, silently, in the header - # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. - alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, - detail=f"{sharer} shared this with you", actor=sharer, - st=session.runtime) - except Exception: # noqa: BLE001 - return - - -def _object_ref(session, kind, oid): - """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. - - ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT - WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced - `target: {module: "database", id: "view_42"}` — an instruction to open a database named - `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not - addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to - travel. Caught by looking at the notification the driver actually produced, not by reading - the code back. - - ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the - receiver nothing they can act on, and the id is already in the target. - ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than - a row that opens nowhere. - """ - try: - if kind == "field": - # ⭐⭐ W38-T16 — A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it - # is a column INSIDE a database, so the target that travels is the DATABASE. Without - # this branch the function falls through to the view/folder loop, finds nothing, - # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and - # the receiver is never told, which is the silent half of owner item 18 reopened one - # kind over. - from routes_alerts import route_for_topic - table_key, field_key = shares.split_field_oid(oid) - if not table_key: - return ("A column", None, "") - try: - defn, _shared, _workspace, shared_key, _grant_topic = _field_definition( - session, table_key, field_key) - except Exception: # noqa: BLE001 - defn = None - label = str((defn or {}).get("label") or "").strip() or field_key - # ⚠ TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed - # by whatever the calling door already held: a `ut_*` database uses its bare key, - # while a registry topic uses `_table_workspace` (`product_data.TABLE_KEY`). - # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not - # `customer_data`), so the suffix comes off before it is asked — rather than a second - # route table being written here, which is how the two come apart. - _WS = "_table_workspace" - scope = {"customer_data": "customer", "product_data": "product"}.get(table_key) - if scope is None: - scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key - return (label, route_for_topic(scope) or None, "") - if kind == "database": - import core.user_tables as ut - defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} - # A user table IS its own route key in both vocabularies (`route_for_topic`). - return (str(defn.get("label") or "").strip() or "A database", str(oid), "") - import core.table_store as table_store - from routes_alerts import route_for_topic - for topic in _topics(session): - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - if not hit: - continue - route = route_for_topic(topic) - if not route: - break - row = hit[1] if len(hit) > 1 else {} - name = str((row or {}).get("name") or "").strip() - # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens - # the grid and stops there rather than naming a view the receiver did not get. - return (name or ("A view" if kind == "view" else "A folder"), - route, str(oid) if kind == "view" else "") - except Exception: # noqa: BLE001 - pass - return ({"view": "A view", "folder": "A folder", - "field": "A column"}.get(kind, "An item"), None, "") +"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). + + GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} + PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) + PUT /api/v1/share/{kind}/{oid}/owner <- {owner: ""} (R6d — `field` ONLY) + GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} + +`kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the +view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE +vocabulary in the UI (R10: "the same picker views use"). + +⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `":"`** +(`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen — +so a grant stored under one would admit the grantee to every `notes` column in the tenant at +once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not +obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator +is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None, +`_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item +18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for +anything that is not a view. + +⭐⭐ **A "Can edit" GRANTEE MAY RE-SHARE A VIEW — OWNER RULING R4, BUILT AS W40-T02** (instruction +4: *"Edit View so a member can share a View as well, not just an admin"*). This REVERSES the flat +owner-or-admin sentence that stood here, and the reversal is bounded three ways, all enforced HERE +and not in the client: + + 1. **ONLY `kind='view'`.** `shares.RESHARE_KINDS` is the one spelling of that. `folder`, + `database` and `field` still require the owner or an admin, and the paragraph below is why + the `database` kind in particular was never a candidate: a `ut_*` grant's blast radius is a + whole table, where a view is one saved SELECTION over rows the receiver's own wall governs. + 2. **A RE-SHARE MAY NEVER EXCEED THE RE-SHARER'S OWN ROLE**, and the strict reading ships: an + `edit` grantee may hand out `view` and NEVER `edit`. `shares.max_grantable_role` answers the + ceiling, `put_share` enforces it on the DELTA (a name arriving at `edit`, or one raised to + it) — never on every row of the body, because the `PUT` REPLACES and `ShareDialog.save` + therefore re-sends the whole list with the re-sharer's own `edit` row inside it. Conferring + `edit` stays the owner's alone. + 3. **OWNERSHIP NEVER MOVES.** A caller re-sharing rather than owning passes the EXISTING owner + straight through (`put_share` below, stated rather than incidental). + +⭐⭐ **AND R4 BOUNDED ONLY THE ROLE. TWO MORE BOUNDS ARRIVED WITH THE OWNER'S RULING OF +2026-08-24, BECAUSE THE GAPS WERE MEASURED ON THE SHIPPED BUILD:** + + 4. **A RE-SHARER MAY SHARE ONLY WITH SPECIFIC PEOPLE (D-473).** Owner: *"a re-sharer can only + share to specific people"*. Before this, `[fisch:edit, *:view]` from an `edit` grantee was a + `200` — the role ceiling could not see it, because `*` at `view` never exceeds a `view` + ceiling. `_audience_added` refuses an `EVERYONE` entry this caller is ADDING, and lets one + the OWNER already placed ride along untouched, for the same delta reason as (2). + 5. **A RE-SHARER MAY UN-SHARE ONLY THE PEOPLE THEY SHARED TO (D-474).** Owner: *"a re-sharer + can only unshare the people it shared to"*. Before this, an `edit` grantee PUTting a list + that omitted another grantee got `200` and that person's access was gone; `PUT []` left the + view shared with nobody. Answering this needs a fact the registry did not store, so each + entry now carries `by` — who placed it — stamped by `shares.set_grants(granter=…)` and + consulted by `_unremovable`. ⛔ An entry with NO `by` (every grant predating the change) is + the owner's or an admin's to revoke and nobody else's: fail-closed, because from the door + "unstamped" and "somebody else's" are the same observation. + +⛔ **SO BOTH FEARS THE OLD SENTENCE NAMED ARE NOW ANSWERED, AND THE PARAGRAPH THAT PRICED THE +SECOND ONE IS GONE RATHER THAN SOFTENED.** It read *"'Widen a users-scoped view to everyone' is +NOT closed — an `edit` grantee may add `*` — it is CAPPED at Can view"*, and that was an accurate +account of R4 that the owner overruled the moment it was put in front of them. *"Grant themselves +ownership and lock you out"* stays closed by (3) and `set_grants`' sticky owner. A `view` grantee +still cannot share at all, and still gets `403`. The client greys the editor for +non-administrators and offers "Can edit" to anyone it greys in; that is a courtesy, and these +checks are the wall. + +⚠ **WHAT `GET` NOW DISCLOSES.** The grant record carries `by`, and this route returns the record +verbatim — so anybody who may read a view's grant list also learns who added each person, a +`view` grantee included. Accepted deliberately (the alternative is a second, stripped read path +and therefore a second answer to one question), and stated here because this file states costs +rather than leaving them to be discovered. + +⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means +every account that can already open the surface: `require_session` plus the topic's own gate run +first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field +closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that +could already reach the data ([[aios-permissioning]]). + +⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED +MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` +and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be +DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes +`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded +by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every +column — and an `*` database grant admits every account in the tenant to all of it. +That is a real capability, deliberately kept; what was wrong was a docstring promising a second +wall that does not exist for this kind. Scoping user tables is booked, not done +(`waves/wave32/sharing-audit.md` S-8). + +⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS +registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` +— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here +whose object is invisible under that one is a row in a list that opens a refusal, which is what +made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but +the two vocabularies are still two. +""" +from fastapi import APIRouter, Body, Depends + +import core.shares as shares +import core.users as users +from deps import Session, err, require_session +# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES +# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the +# reader agreeing about one string is the whole difference between an Inbox row that opens the +# shared database and one that is quietly unclickable. +from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC + +router = APIRouter(prefix="/api/v1") + + +def _kind_or_400(raw): + try: + return shares._check_kind(raw) + except ValueError as e: + raise err(400, "bad_kind", str(e)) + + +# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── +# +# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all +# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free +# strings off the URL and the only dependency was `require_session`, so any signed-in account +# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, +# an object with no grant record skipped the check entirely and the caller was stamped OWNER — +# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not +# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. +# +# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own +# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the +# victim next opens the dialog. + +#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share +#: route is not told which topic — so resolving one means asking each. +_BUILTIN_TOPICS = ("customer", "product") + + +def _field_storage_keys(table_key): + """Resolve the client-facing field topic to its durable stores and grant topic.""" + raw = str(table_key or "").strip() + if raw in ("customer_data", "customer_table_workspace"): + return "customer_table_workspace", "customer_table_workspace", "customer_data" + if raw in ("product_data", "product_table_workspace"): + return "product_table_workspace", "product_table_workspace", "product_data" + if raw.startswith("ut_"): + bare = raw[:-len("_table_workspace")] if raw.endswith("_table_workspace") else raw + return f"{bare}_table_workspace", bare, bare + workspace = raw if raw.endswith("_table_workspace") else f"{raw}_table_workspace" + shared = raw[:-len("_table_workspace")] if raw.endswith("_table_workspace") else raw + return workspace, shared, shared + + +def _field_definition(session, table_key, field_key): + """Return the shared/private definition and the keys used by its write paths.""" + workspace_key, shared_key, grant_topic = _field_storage_keys(table_key) + try: + from core import shared_overlay + shared = (shared_overlay.fields(shared_key, st=session.runtime) or {}).get(field_key) + if isinstance(shared, dict): + return shared, True, workspace_key, shared_key, grant_topic + import core.table_store as table_store + private = (table_store.make(workspace_key, st=session.runtime) + .workspace(session.uname).get("fields") or {}).get(field_key) + if isinstance(private, dict): + return private, False, workspace_key, shared_key, grant_topic + except Exception: # noqa: BLE001 + pass + return None, False, workspace_key, shared_key, grant_topic + + +def _field_owner(session, definition, already_shared): + """Resolve the creator for the share claim wall. + + A private field is already namespaced by the caller's own workspace. Older field records + from before the host-side creator stamp therefore remain safely claimable by that workspace + owner, while a shared definition with no creator stays admin-only because its storage is + tenant-wide and cannot identify an owner from residency alone. + """ + owner = str((definition or {}).get("createdBy") or "").strip() + if owner: + return owner + return str(session.uname or "").strip() if not already_shared else "" + + +def _topics(session): + """Every topic whose workspace could hold a view or folder for this tenant. + + ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on + tenant #0) to answer a question about KEYS (D-185). + """ + try: + import core.user_tables as ut + return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) + except Exception: # noqa: BLE001 + return _BUILTIN_TOPICS + + +def _owns_object(session, kind, oid): + """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? + + ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking + each topic's workspace in turn, which is N store reads; making every share call pay that + would put a loop on a route the manage-access dialog opens. The dangerous path is the one + where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs + exactly there, and the common path (a record exists, `may_administer` decides) is untouched. + """ + if session.admin: + return True + if kind == "field": + # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS + # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 / + # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be + # deletable by one person and shareable by another. + # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION: + # a brand-new column has no grant record, so `put_share` falls to this predicate — and + # without it the column's own creator is answered `404 no_object` on the first attempt to + # share the thing they just made. + table_key, field_key = shares.split_field_oid(oid) + if not table_key: + return False + defn, _shared, _workspace, _shared_key, _grant_topic = _field_definition( + session, table_key, field_key) + owner = _field_owner(session, defn, _shared) + return bool(defn) and owner.lower() == str(session.uname).strip().lower() + if kind == "database": + # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already + # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" + # here would be the second definition this wave keeps finding. + try: + import core.user_tables as ut + return bool(ut.may_open(oid, session.uname, is_admin=session.admin, + st=session.runtime)) + except Exception: # noqa: BLE001 + return False + try: + import core.table_store as table_store + except Exception: # noqa: BLE001 + return False + for topic in _topics(session): + try: + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + except Exception: # noqa: BLE001 + continue + if hit: + # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the + # person whose personal stratum holds it — anybody else reaching this line is + # exactly the case S-1 describes. + return str(hit[0]) == str(session.uname) + return False + + +def _can_see_object(session, kind, oid): + """May this caller READ an object's grant list — i.e. can they reach the object at all? + + ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A + REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the + ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches + PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with + `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit + in the grid** — answered `404` when bob opened its manage-access dialog. Measured before + fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said + `404 no_object`. + ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", + and the wall consulted the grant registry (system A) plus stratum ownership, never the view's + `permissions` (system B) — which is the one that actually decides who may OPEN it. + ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not + gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — + 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a + legitimate collaborator their view does not exist. + + ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its + owner — that is S-1, and widening this predicate onto `put_share` would re-open it. + """ + if _owns_object(session, kind, oid): + return True + if kind != "view": + # A folder carries no per-object visibility flag of its own, and a database's `may_open` + # (inside `_owns_object`) already admits grantees. Nothing wider to ask. + # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here: + # `get_share` tests `role is None` first and a grant answers a role, so the only caller + # left is an account with no relationship to the column at all. Widening it would let any + # signed-in session enumerate who holds which column on a database they cannot open. + return False + try: + import core.table_store as table_store + for topic in _topics(session): + hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) + if hit: + return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, + session.uname, is_admin=session.admin)) + except Exception: # noqa: BLE001 + return False + return False + + +def _entries_or_400(session, entries, refusal=None): + """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. + + ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: + a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE + of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, + so a typo'd name is stored, reported as a successful save, and never reaches anybody. The + sharer believes the person has access. That is item 18's plain reading. + ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker + (`_people`). One route, two populations, and the write door was the permissive one. + ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for + "every account that can already open the surface". ⛔ A CALLER THAT NEEDS ONE REAL PERSON MUST + THEREFORE REFUSE `*` ITSELF, BEFORE ASKING HERE — see `reassign_owner`, where "everyone" is + not an answer to "who owns this column". + + ⭐ W41-T03 — `refusal` IS THE SENTENCE AFTER THE NAME, AND ITS DEFAULT IS BYTE-IDENTICAL to + what this function has always printed. The tenant-registry check is the part that must not be + written twice ([[one-evaluator-per-question]]); the advice at the end of it is the part that + has to match the door the reader is standing at, because "Nothing was shared" is the wrong + account of a refused OWNER change. One validator, one population, one code (`unknown_people`, + which is what clients match on) — and a tail the caller owns. + """ + known = {p["username"].strip().lower() for p in _people(session.tenant)} + unknown = [] + for e in entries or (): + if not isinstance(e, dict): + continue + user = str(e.get("user") or "").strip().lower() + if user and user != shares.EVERYONE and user not in known: + unknown.append(user) + if unknown: + raise err(400, "unknown_people", + "no active account in this workspace is named " + + ", ".join(sorted(set(unknown))) + ". " + + (refusal or "Nothing was shared. Pick people from the list rather than " + "typing a name.")) + + +def _named(who): + """The people in a refusal, in the words the person reading it uses. + + ONE spelling, used by all three re-share refusals below. `*` is never printed raw: the store's + wildcard is a single character, and a 403 reading *"set * to Can view"* names nothing a person + can find in the dialog they are looking at. + """ + return ", ".join("everyone in this workspace" if w == shares.EVERYONE else w + for w in sorted(who)) + + +def _audience_added(entries, held): + """⭐⭐ D-473 — the `EVERYONE` grant this caller is ADDING, or the empty set. + + OWNER RULING 2026-08-24, verbatim: *"a re-sharer can only share to specific people"*. R4's + ceiling bounded the ROLE a re-sharer may hand out and said nothing about the AUDIENCE, and the + gap was measured rather than argued: an `edit` grantee PUTting `[fisch:edit, *:view]` was + answered `200`, widening a two-person view to the whole tenant at `Can view`. The role check + could not catch it, because `*` at `view` never exceeds a `view` ceiling. + + ⛔ IT IS THE ADDITION THAT IS REFUSED, NOT THE PRESENCE, and that is the same shape as the + role check one arm above, for the same reason: the `PUT` REPLACES, so `ShareDialog.save` + re-sends the WHOLE list every time. An `*` the OWNER placed rides along in every payload the + re-sharer is able to produce, and refusing on presence would `403` every save on a view the + owner had already opened to everyone — a re-sharer locked out of a list they may legitimately + edit, with a message about a row they did not touch. + + ⛔ ROLE IS NOT CONSULTED HERE, DELIBERATELY. This answers "may this caller widen the + AUDIENCE", and `*` held at `view` and resubmitted at `edit` is a ROLE escalation that the + check above already refuses, by name. Two questions, two predicates + ([[one-evaluator-per-question]]) — and separable is also what lets a gate disarm one of them + in memory and prove the other still fires. + + ⚠ `_clean_entries`, NEVER THE RAW BODY: an entry with a junk role is dropped by the writer, + so reading the raw list would refuse a widening that was never going to be stored. + """ + if shares.EVERYONE in (held or {}): + return set() + return {e["user"] for e in shares._clean_entries(entries) if e["user"] == shares.EVERYONE} + + +def _unremovable(held_rows, entries, uname): + """⭐⭐ D-474 — the people this caller is dropping from the list but may NOT revoke. + + OWNER RULING 2026-08-24, verbatim: *"a re-sharer can only unshare the people it shared to"*. + R4 bounded what a re-sharer may HAND OUT and left what they may TAKE AWAY unbounded, and both + halves of that were measured: an `edit` grantee PUTting a list that omits another grantee was + answered `200` and that person's access was gone; `PUT []` left the view shared with nobody. + + ⛔⛔ AN ENTRY WITH NO `by` IS NOT REMOVABLE BY A RE-SHARER — ONLY BY THE OWNER OR AN ADMIN, + AND THAT IS THE FAIL-CLOSED DIRECTION RATHER THAN AN OVERSIGHT. Every grant written before + provenance existed carries no stamp, so "no `by`" and "granted by somebody else" are + indistinguishable from here. Reading absence as *"nobody claims it, so anyone may take it"* + would hand every re-sharer the power to revoke the entire pre-existing grant set on day one + of this change, which is the exact capability the ruling withholds. A re-sharer must not be + able to revoke a grant they cannot PROVE they made [[aios-permissioning]]. + + ⚠ SO A RE-SHARER CANNOT REMOVE THEMSELVES EITHER, AND THAT IS STATED BECAUSE IT LOOKS LIKE A + BUG. Their own row was placed by the owner, so it carries the owner's `by` and lands in this + set. Read literally, the ruling says a re-sharer unshares only who THEY shared to, and their + own grant is not one of those. Leaving the view is the owner's to do, like every other + revocation of an owner-placed grant. ⛔ Do not carve an exception here without a ruling: the + carve-out is indistinguishable from "a re-sharer may revoke any row whose `by` names the + owner", which is the wall itself. + + ⚠ `_clean_entries`, NEVER THE RAW BODY, AND THIS IS THE HOLE THAT SHAPE CLOSES. A role the + writer rejects is a row that will NOT be stored, so `{user: victim, role: "nonsense"}` looks + present in the raw payload and is a silent REVOCATION once written. Asking the same + normaliser the store uses is what makes "submitted" mean the same thing at both ends. + """ + me = str(uname or "").strip().lower() + submitted = {e["user"] for e in shares._clean_entries(entries)} + stuck = set() + for user, row in (held_rows or {}).items(): + if user in submitted: + continue + stamp = str((row or {}).get("by") or "").strip().lower() + if not me or not stamp or stamp != me: + stuck.add(user) + return stuck + + +@router.get("/share/mine") +def my_shares(session: Session = Depends(require_session)): + """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). + + Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves + in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL + that is not malformed at all. + """ + return shares.shared_with(session.uname, st=session.runtime) + + +@router.get("/share/{kind}/{oid}") +def get_share(kind: str, oid: str, session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + if kind == "field": + table_key, field_key = shares.split_field_oid(oid) + if table_key and field_key: + oid = shares.field_oid(_field_storage_keys(table_key)[2], field_key) + rec = shares.grants(kind, oid, st=session.runtime) + role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime) + # W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant + # registry has no owner to return from `may_administer`, even though the same caller may safely + # claim their own object through the PUT path below. Reflect that exact claim predicate here: + # a Member who owns an unshared View receives the people picker; a collaborator still does not. + if not rec["owner"] and not may_admin and _owns_object(session, kind, oid): + may_admin = True + # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: + # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — + # to any signed-in session, about objects it cannot open. Now a caller with no role on an + # object must prove they can reach it, and gets a 404 otherwise: the same answer a + # non-existent id gives, so the route cannot be used to probe which ids are real. + # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a + # caller who has no relationship with the object at all. + if role is None and not _can_see_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + return { + **rec, + "role": role, + "mayAdminister": may_admin, + # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry + # them. `assignable_people` serves bare display names because `user`-kind CELLS store + # display names — that list's shape cannot change without migrating cell values — so + # this route serves objects of its own. Existing grants that were written as lowercased + # display names are normalised by the wave-21 cleanup script. + # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller + # who may open the editor. A read-only grantee gets the grant list (their fair question is + # "who else has this?") and not a directory of every account in the workspace. + # ⭐⭐ W40-T02 / R4 — AND THAT RULE IS WHY THIS LINE NEEDED NO EDIT. `may_administer` now + # answers True for an `edit` grantee on a VIEW, which MOVES that account into "may open + # the editor" — so the picker they need arrives by the roster riding on the same flag it + # always did. Gating it on anything else (owner, `role == 'owner'`, a fresh predicate) + # would be a second answer to a question this file already answers once, and would leave + # the new grantee with an editor and no people to put in it. A `view` grantee is still + # `may_admin=False` here and still gets `[]`. + "people": _people(session.tenant) if may_admin else [], + } + + +def _people(tenant): + """[{username, name}] for this tenant — same population as `assignable_people`, with the + BINDING identity alongside the display one.""" + try: + reg = users.registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + out = [] + for uname, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append({"username": str(uname), "name": str(u.get('name') or uname)}) + return sorted(out, key=lambda p: p["name"].lower()) + + +@router.put("/share/{kind}/{oid}") +def put_share(kind: str, oid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + if kind == "field": + table_key, field_key = shares.split_field_oid(oid) + if table_key and field_key: + oid = shares.field_oid(_field_storage_keys(table_key)[2], field_key) + body = body or {} + rec = shares.grants(kind, oid, st=session.runtime) + # `claiming` is the "no owner yet, and this caller may become one" branch, hoisted to a name + # because TWO decisions below need it: the ceiling (a claimant is about to be the owner, so + # their ceiling is an owner's) and the owner written back (D3 — see `set_grants` at the end). + claiming = False + # An object with NO grant record yet has no owner — the first person to share it claims it. + # That is safe because reaching this route at all means passing the surface's own wall, and + # the alternative (refusing until somebody seeds an owner) would make a brand-new folder + # unshareable by the person who just made it. + if rec["owner"]: + if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime): + # ⭐ R4 / W40-T02 — `may_administer` now also admits an `edit` grantee on a VIEW, so + # the population refused here is narrower than the code word `not_owner` suggests: a + # `view` grantee, or an account with an `edit` role on a kind outside + # `shares.RESHARE_KINDS`. The code string is kept because clients match on it. + raise err(403, "not_owner", + "only the owner of this item (or an administrator) can change who it is " + "shared with") + # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record + # is still claimed by the first person to share it — that rule is right, and refusing until + # somebody seeds an owner would make a brand-new folder unshareable by the person who just + # made it. What was missing is the half the old comment ASSERTED and the code never did: the + # claimant has to be able to reach the object. Without this, any signed-in account could + # stamp itself owner of an id it had never seen and lock the real creator out for good. + elif not _owns_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + else: + claiming = True + entries = body.get("entries") + if not isinstance(entries, list): + raise err(400, "bad_entries", + "entries must be a list of {user, role}. Send [] to un-share, which is how " + "revoking is expressed") + _entries_or_400(session, entries) + # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit` + # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer + # holds") needs a check of its own: that caller may hand out `view`, and conferring `edit` + # stays the owner's or an administrator's. + # + # ⛔ AFTER THE ADMISSION, NEVER BEFORE, AND THAT ORDER IS A SECURITY PROPERTY. A caller with + # no role at all must keep receiving `404 no_object` (audit S-1/S-3: a stranger learns + # nothing, so this route cannot be used to probe which ids are real). A ceiling raised first + # would answer that caller `403` and turn the one route hardened against id-probing back into + # an oracle that confirms an id exists. It also runs before the `field` promotion below, so a + # refusal cannot leave a column promoted with no grant written. + # + # ⛔ AND IT IS THE DELTA, NOT EVERY ROW OF THE BODY — read off the shipped client, not + # assumed. `ShareDialog.save` PUTs the WHOLE list every time ("a body assembled from a delta + # would revoke everyone it failed to mention"), so the re-sharer's OWN `{user, role: "edit"}` + # row rides in every payload they are able to produce. Refusing per-entry would `403` the + # exact re-share this ticket exists to enable, and the only body that would pass is one that + # revokes the re-sharer. So what is refused is edit access this caller is CREATING: a name + # arriving at `edit`, or an existing `view` grantee raised to it. A row that already stood at + # `edit` was the OWNER's decision, and is not this caller's to be refused for. + # + # ⚠ A CLAIMANT IS AN OWNER. The branch above admits a Member who owns an object that has no + # grant record yet, and `set_grants` is about to stamp them owner — asking the registry for + # their role here would answer `None` (no record exists to hold one) and refuse the first + # `edit` grant on every newly created view. Same predicate as the door, one line apart. + ceiling = "edit" if claiming else shares.max_grantable_role( + kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + # + # ⭐⭐ OWNER RULING 2026-08-24 (D-473 + D-474) — AND THE CEILING IS NOW ONE OF THREE WALLS IN + # THIS BLOCK, NOT THE WALL. R4 bounded the ROLE a re-sharer may hand out and was silent on the + # other two questions a re-share asks, so both gaps shipped and both were measured on the + # build: an `edit` grantee could PUT `[fisch:edit, *:view]` and widen a two-person view to the + # whole tenant (200), and could PUT a list omitting another grantee — or `[]` — and revoke + # people they never granted (200). The owner's answer settles both in one sentence: *"no a + # re-sharer can only share to specific people and a re-sharer can only unshare the people it + # shared to"*. + # + # ⛔ THREE PREDICATES, THREE FUNCTIONS, ONE ORDER: role, then audience, then revocation. They + # are separate because they answer separate questions and because a wall that cannot be + # disarmed ALONE cannot be proven alone — `verify_scopes.section_reshare_bounds` patches each + # one in memory and requires exactly its own leg to go red, which a single fused `if` would + # make impossible ([[a-declared-gate-is-an-unchecked-claim]]). + # ⚠ ROLE AND AUDIENCE COMPOSE, AND THE ORDER DECIDES WHICH REFUSAL A PERSON READS. `*` + # submitted at `edit` while held at `view` is BOTH an escalation and (if unheld) a widening; + # the role check runs first and names the fix that is actually available to this caller + # ("set it to Can view"), which is the more useful of the two sentences. + if ceiling != "edit": + # The prior ROWS, not just their roles: the revocation wall needs each entry's `by`, and + # reading it from a second place would be a second answer to "what does the store hold". + held_rows = {e.get("user"): e for e in (rec["entries"] or ()) + if isinstance(e, dict) and e.get("user")} + held = {u: e.get("role") for u, e in held_rows.items()} + noun = {"view": "view", "folder": "folder", + "database": "database", "field": "column"}.get(kind, "item") + raised = set() + for e in entries: + if not isinstance(e, dict): + continue + who = str(e.get("user") or "").strip().lower() + if who and str(e.get("role") or "").strip().lower() == "edit" \ + and held.get(who) != "edit": + raised.add(who) + if raised: + raise err(403, "grant_exceeds_role", + "you can share this " + noun + " at Can view, which is as far as your own " + "access reaches. Only its owner (or an administrator) can give somebody " + "Can edit, so nothing was saved. Set " + _named(raised) + + " to Can view and save again.") + # ⭐⭐ D-473 — THE AUDIENCE. A re-sharer names PEOPLE; reaching "everyone" is the owner's. + if _audience_added(entries, held): + raise err(403, "grant_exceeds_audience", + "you can share this " + noun + " with specific people, which is as far as " + "your own access reaches. Only its owner (or an administrator) can open it " + "to everyone in this workspace, so nothing was saved. Remove Everyone from " + "the list, add the people you meant by name, and save again.") + # ⭐⭐ D-474 — THE REVOCATION. An omission IS a revocation on a replacing PUT, so this is + # the only place a removal can be refused. ⛔ REFUSED WHOLE: `set_grants` has not run, so + # a payload carrying a legitimate addition ALONGSIDE a forbidden removal saves neither. + # That is deliberate and it is what the message promises ("nothing was saved") — a + # half-applied permission change is worse than a refused one, because the person reading + # the toast has no way to tell which half took. + stuck = _unremovable(held_rows, entries, session.uname) + if stuck: + raise err(403, "revoke_not_yours", + "you can remove the people you shared this " + noun + " with, and this " + "workspace has no record of you sharing it with " + _named(stuck) + + ". Only its owner (or an administrator) can remove them, so nothing was " + "saved. Put them back on the list and save again.") + if kind == "field": + # A field grant is a visibility and edit wall. Promote a private custom + # field exactly once, then keep the requested Share field role as the + # authoritative override for the legacy permissions bag. + from core import field_permissions, shared_overlay + table_key, field_key = shares.split_field_oid(oid) + defn, already_shared, workspace_key, shared_key, grant_topic = _field_definition( + session, table_key, field_key) + if not isinstance(defn, dict): + raise err(404, "no_object", "no such field, or it is not shared with this account") + if not already_shared: + defn = field_permissions.promote_field( + workspace_key, shared_key, grant_topic, + session.uname, defn, st=session.runtime) + stamped = dict(defn) + stamped["shared"] = True + stamped["granted"] = True + shared_overlay.put_field(shared_key, field_key, stamped, st=session.runtime) + oid = shares.field_oid(grant_topic, field_key) + # ⭐⭐ R4 / W40-T02 (D3) — OWNERSHIP NEVER MOVES ON A RE-SHARE, AND IT IS SAID HERE RATHER + # THAN LEFT TO FALL OUT. `set_grants`' owner is sticky, so the old `rec["owner"] or + # session.uname` already happened not to transfer ownership — incidentally, as a property of + # the callee. R4 names ownership transfer as one of the two halves of the old protection that + # SURVIVES the widening, and a rule that survives by accident is one the next edit deletes + # without noticing. So the branch is explicit: a claimant becomes the owner, and everybody + # else — an owner re-saving, an admin, and now an `edit` grantee re-sharing — passes the + # EXISTING owner straight back through. `claiming` is the same flag the admission set, so + # there is no second answer to "is this person taking ownership". + # ⭐⭐ D-474 — `granter` IS THIS SESSION, ON EVERY SAVE INCLUDING THE OWNER'S. Provenance is + # recorded for whoever adds a person, not only for a re-sharer: an owner-placed grant carrying + # NO stamp is indistinguishable from a pre-provenance one, and `_unremovable` would then be + # deciding on the store's AGE rather than on who granted what. `set_grants` stamps only + # entries that are NEW to the record and never re-stamps an existing one, so an owner + # re-saving a list does not quietly take provenance off the re-sharer who built it. + # ⛔⛔ THE THREE REFUSALS ABOVE WERE DECIDED AGAINST `rec`, WHICH WAS READ AT THE TOP OF + # THIS FUNCTION. Handing `expect` to the writer is what makes them true at the moment of the + # write rather than at the moment of the read: a concurrent save lands between the two, and + # a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the + # owner had just added disappeared. See `shares.set_grants`' own note. + try: + out = shares.set_grants(kind, oid, entries, + owner=session.uname if claiming else rec["owner"], + granter=session.uname, + st=session.runtime, + expect=rec["entries"]) + except shares.GrantsChanged as exc: + raise err(409, "grants_changed", str(exc)) + _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) + return out + + +@router.put("/share/{kind}/{oid}/owner") +def reassign_owner(kind: str, oid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """⭐⭐ W41-T03 / RULING R6(d) — HAND ONE COLUMN TO A NEW OWNER. `{"owner": ""}`. + + ⛔ A SEPARATE DOOR, NOT AN `owner` MEMBER ON `PUT /share/{kind}/{oid}`, AND THE REASON IS THE + FAILURE MODE RATHER THAN TIDINESS. `GET` answers the stored record — `{owner, entries, ...}` — + and `ShareDialog` parses the whole of it into its state. The moment any client re-serialises + that state on save (today `shareModel.sharePutBody` returns `{entries}` alone, which is one + refactor away from `{...state}`), an `owner` member on the share body would REASSIGN THE COLUMN + because the client echoed back a key it had always been given. A door nobody can walk through + by accident is the whole point [[a-guard-authorises-a-destination-not-a-payload]]. + + ⛔ AND THE TWO DOORS DO NOT SHARE A WALL, WHICH IS THE SECOND AND STRONGER REASON. `put_share` + admits whoever `shares.may_administer` admits, and since R4 that INCLUDES an `edit` grantee on + a view. Ownership is the owner's and an administrator's alone, so this asks + `shares.role_for(...) == 'owner'` instead. Fusing the two would put two different walls behind + the presence of one key in one body ([[one-evaluator-per-question]]). + + ⭐⭐ `field` ONLY, AND THE REFUSAL FOR THE OTHER THREE KINDS IS A REAL FINDING RATHER THAN + SCOPE-TRIMMING. This registry's `owner` is AUTHORITATIVE for a column: `field_permissions. + field_class` (contract C1) reads it and falls back to `createdBy` only when there is no record, + and `user_tables.delete_field_refusal` then derives R6's delete right from that bag. For the + other three kinds it is NOT: a view or folder is found by `table_store.find_view`/`find_folder` + searching PERSONAL STRATA, and a database resolves through `user_tables.may_open` — so moving + the registry owner there would move the badge and leave both the rights and the residency + behind, i.e. ship the half that is visible and not the half that is true. That is a bigger + ticket than this one, so this door says no rather than pretending. + + ⚠ ORDER IS A SECURITY PROPERTY HERE, EXACTLY AS IN `put_share`: the wall runs BEFORE the new + owner is validated, because validating a username consults `_people` — the tenant's whole + account directory — and a stranger must not be able to use this route as a roster oracle or an + id oracle (audit S-1/S-3). A caller with no relationship to the column keeps getting the same + `404` a made-up id gets. + + ⚠ AND THE PREVIOUS OWNER IS NOT KEPT ON THE LIST. Reassignment moves the seat; it does not + leave a consolation grant behind, because W41-T03's `done-when` requires the previous owner to + STOP passing `may_delete_field`. An administrator who wants them to keep access adds them as an + ordinary entry through the share door, which is the one place grants are decided. + """ + kind = _kind_or_400(kind) + if kind != "field": + raise err(400, "kind_not_reassignable", + "only a column's owner can be changed here. A view, a folder and a database " + "each record their owner outside this list, so nothing was changed.") + table_key, field_key = shares.split_field_oid(oid) + if table_key and field_key: + oid = shares.field_oid(_field_storage_keys(table_key)[2], field_key) + rec = shares.grants(kind, oid, st=session.runtime) + # ⛔⛔ THE WALL: AN ADMINISTRATOR OR THE CURRENT OWNER, AND NOBODY ELSE. `role_for` answers + # `'owner'` for both (an admin reads as owner by its own rule), and it answers `'edit'` — not + # `'owner'` — for a grantee. That distinction is load-bearing on this kind in particular: + # `field_permissions.migrate_legacy_fields` writes `[{user: '*', role: 'edit'}]` on every + # promoted legacy column, so a wall spelled `may_edit` or `may_administer` would hand EVERY + # account in the tenant the right to reassign every migrated column. + role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + if rec["owner"]: + if role != "owner": + raise err(403, "not_owner", + "only the owner of this column (or an administrator) can hand it to " + "somebody else, so nothing was changed.") + # ⚠ NO RECORD YET MEANS THE OWNER IS THE COLUMN'S CREATOR, and `_owns_object` is the one place + # that reads `createdBy` for this question — the same predicate `put_share` claims through, one + # function apart. A stranger falls out here as `404`, never `403`: a `403` would confirm the id + # is real, which is the oracle S-3 closed. + elif role != "owner" and not _owns_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + # ⛔ FAIL CLOSED ON A COLUMN THIS DOOR CANNOT RESOLVE, AND NAME WHICH HALF FAILED. + defn, already_shared = None, False + if table_key and field_key: + defn, already_shared = _field_definition(session, table_key, field_key)[:2] + if not isinstance(defn, dict): + raise err(404, "no_object", + "this column could not be found on that database, so its owner was not changed.") + # ⛔⛔ A PRIVATE COLUMN CANNOT BE HANDED OVER, AND REFUSING IS THE ONLY HONEST ANSWER. An + # unpromoted definition lives in ONE account's own workspace stratum, so a reassignment would + # write a registry owner who can never see the column while the creator — no longer the + # registry owner — loses the share door on it. That is precisely the unmanageable object + # `shares.set_grants`' sticky-owner note exists to prevent, arriving through a new door. + # ⛔ AND THE FIX IS NOT TO PROMOTE IT HERE. `put_share` promotes because sharing is what the + # caller asked for; promotion stamps the definition tenant-wide and would move contract C1's + # `audience` badge from `private` to `everyone`. An ownership change must never widen an + # audience as a side effect. + if not already_shared: + raise err(409, "field_not_shared", + "this column is still private to the person who made it, so it cannot be handed " + "to somebody else yet. Share it first, then change its owner.") + body = body or {} + new_owner = str(body.get("owner") or "").strip().lower() + if not new_owner: + raise err(400, "bad_owner", + "name the account that should own this column. Nothing was changed.") + # ⚠ `_entries_or_400` ADMITS `*` DELIBERATELY (it is R10's word for "everyone"), so the one + # refusal it cannot make for us is made here: a column is owned by a PERSON, and an owner of + # `*` is an owner nobody can log in as. + if new_owner == shares.EVERYONE: + raise err(400, "bad_owner", + "a column is owned by one person, not by everyone in this workspace. Pick an " + "account by name. Nothing was changed.") + # ⛔ THE SAME VALIDATOR THE SHARE DOOR USES, NEVER A SECOND ONE. A reassignment to a name no + # active account in this tenant answers to would strand the column with an owner who cannot + # sign in — the ownership-shaped version of exactly what item 18 was about. + _entries_or_400(session, [{"user": new_owner, "role": "view"}], + refusal="Nothing was changed. Pick the new owner from the list rather than " + "typing a name.") + # ⭐⭐ D-474 — `granter` IS DELIBERATELY NOT PASSED, AND THAT IS NOT AN OMISSION. This call + # adds NOBODY: it writes back the entry list exactly as it was read, so every user is already + # in the prior record and `set_grants` keeps each stored `by` VERBATIM. Naming a granter would + # be inert on that path today and WRONG on the rebase path `set_grants` warns about — where a + # user reclassified from "prior" to "new" would silently have their provenance transferred to + # whoever reassigned the column, handing an administrator the right to revoke people a + # re-sharer had granted. Absent, such a user gets no stamp at all, which is the fail-closed + # direction `_unremovable` already reads as "the owner's alone to revoke". + # ⛔ `expect` IS PASSED FOR THE REASON `put_share` PASSES IT: this write REPLACES the entry set + # with the snapshot read at the top of this function, so a grant somebody added in between + # would be silently deleted by an operation that is supposed to touch only the owner. The + # compare-and-set turns that into a 409 the person can act on. + # ⚠ WHAT `expect` DOES NOT COVER IS THE OWNER ITSELF — it compares `(user, role)` pairs only. + # Two concurrent reassignments therefore both land, last write winning. Both callers held the + # seat when they decided, so this is a lost update between authorised callers rather than an + # escalation; widening the compare-and-set to the owner is a change to a shared primitive and + # is booked rather than smuggled in here. + try: + return shares.set_grants(kind, oid, rec["entries"], owner=new_owner, + st=session.runtime, expect=rec["entries"]) + except shares.GrantsChanged as exc: + raise err(409, "grants_changed", str(exc)) + + +def _notify_new_grantees(session, kind, oid, before, after): + """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. + + Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was + silent: the grant landed in a rail section the receiver had to notice on their own, which is + why "I shared it with you" and "I never saw it" were both true. + + ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read + because an alert is a live question about rows; a share is an EVENT that happened once, and + polling for it would mean re-deriving "was this new?" on every inbox open — the diff below + only exists here, at the moment the set changes. + + ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), + so every save re-sends everyone who was already there. Diffing against `before` is what stops + a rename or a role change from ringing the bell for people whose access did not change. + ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in + the tenant on a single click is a broadcast nobody asked for. The rail still shows it. + ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the + grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. + """ + try: + was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} + fresh = [str(e.get("user")) for e in (after or ()) + if isinstance(e, dict) and e.get("user") not in was + and e.get("user") != shares.EVERYONE] + if not fresh: + return + import core.alerts as alerts + + label, route, view_id = _object_ref(session, kind, oid) + if not route: + # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and + # `notification_view` would have to invent a target. Silence is the honest answer + # here; the rail still shows the grant under "Shared with me". + return + sharer = str(session.user.get("name") or session.uname) + for user in fresh: + # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must + # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes + # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED + # from there rather than typed again — one vocabulary, one owner. + # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY + # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and + # are honestly named by their machine; a SHARE has a real person, and only this call + # site knows who. ⛔ It is passed as its OWN field rather than recovered from the + # `detail` prose below: a sender parsed out of " shared this with you" breaks + # the first time the sentence is reworded, silently, in the header + # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. + alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, + detail=f"{sharer} shared this with you", actor=sharer, + st=session.runtime) + except Exception: # noqa: BLE001 + return + + +def _object_ref(session, kind, oid): + """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. + + ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT + WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced + `target: {module: "database", id: "view_42"}` — an instruction to open a database named + `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not + addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to + travel. Caught by looking at the notification the driver actually produced, not by reading + the code back. + + ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the + receiver nothing they can act on, and the id is already in the target. + ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than + a row that opens nowhere. + """ + try: + if kind == "field": + # ⭐⭐ W38-T16 — A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it + # is a column INSIDE a database, so the target that travels is the DATABASE. Without + # this branch the function falls through to the view/folder loop, finds nothing, + # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and + # the receiver is never told, which is the silent half of owner item 18 reopened one + # kind over. + from routes_alerts import route_for_topic + table_key, field_key = shares.split_field_oid(oid) + if not table_key: + return ("A column", None, "") + try: + defn, _shared, _workspace, shared_key, _grant_topic = _field_definition( + session, table_key, field_key) + except Exception: # noqa: BLE001 + defn = None + label = str((defn or {}).get("label") or "").strip() or field_key + # ⚠ TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed + # by whatever the calling door already held: a `ut_*` database uses its bare key, + # while a registry topic uses `_table_workspace` (`product_data.TABLE_KEY`). + # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not + # `customer_data`), so the suffix comes off before it is asked — rather than a second + # route table being written here, which is how the two come apart. + _WS = "_table_workspace" + scope = {"customer_data": "customer", "product_data": "product"}.get(table_key) + if scope is None: + scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key + return (label, route_for_topic(scope) or None, "") + if kind == "database": + import core.user_tables as ut + defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} + # A user table IS its own route key in both vocabularies (`route_for_topic`). + return (str(defn.get("label") or "").strip() or "A database", str(oid), "") + import core.table_store as table_store + from routes_alerts import route_for_topic + for topic in _topics(session): + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + if not hit: + continue + route = route_for_topic(topic) + if not route: + break + row = hit[1] if len(hit) > 1 else {} + name = str((row or {}).get("name") or "").strip() + # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens + # the grid and stops there rather than naming a view the receiver did not get. + return (name or ("A view" if kind == "view" else "A folder"), + route, str(oid) if kind == "view" else "") + except Exception: # noqa: BLE001 + pass + return ({"view": "A view", "folder": "A folder", + "field": "A column"}.get(kind, "An item"), None, "") diff --git a/api/routes_slack.py b/api/routes_slack.py index 50d382c1fae50935e6148dda677d9a8772dd110d..a2d7853942165a006f557e79ec53c035def5796a 100644 --- a/api/routes_slack.py +++ b/api/routes_slack.py @@ -1,789 +1,789 @@ -"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a -person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) - -Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel -whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle -permissioning per user."* - -──────────────────────────────────────────────────────────────────────────────────────────────── -⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). - -`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since -2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is -`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own -`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its -authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a -live specimen of identity that LOOKS like a permission and is not. - -`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per -principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` -below, which is the whole projection and is nine lines. Study: -`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). - -What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` -below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a -second Node process, a persistent outbound socket and each tenant's Slack credentials held by a -third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven -unauthenticated-public shape plus Slack's signing-secret check. - -──────────────────────────────────────────────────────────────────────────────────────────────── -⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every -human in that channel reads through the same rules. That is the honest reading of "a bot per -channel" and the pane says it out loud, because the alternative is an admin assuming per-person -scoping from a screen that looks exactly like the per-person one. - -⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. -A wall bound to `#sales` silently re-points the day somebody renames the channel. -""" -import hmac -import json -import os -import secrets -import time -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends, Request - -from deps import Session, err, require_session -from routes_admin import admin_gate - -router = APIRouter(prefix="/api/v1") - -#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides -#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. -AGENTS_KEY = "slack_agents" - -#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. -PENDING_KEY = "slack_pending_writes" - -#: How long an approval card stays clickable. An approval is a statement about the world as it was -#: when the card was rendered; an hour later the values it names may no longer be the values it -#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. -APPROVAL_TTL_S = 15 * 60 - -#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. -SLACK_MAX_SKEW_S = 60 * 5 - -MAX_BODY_BYTES = 64 * 1024 - -#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied -#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: -#: past a certain length Slack collapses a message behind "Show more", and a human clicking -#: Approve on a card whose tail they never saw is worse than no card at all. -APPROVAL_MAX_ROWS = 12 - - -# ── the tenant's agent records ─────────────────────────────────────────────────────────────── -def _agents(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this tenant has no agents", never to a 500 on the settings pane.""" - try: - found = rt.get(AGENTS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _clean_agent(raw, prior=None): - """One stored record, built KEY BY KEY. - - ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; - a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule - `routes_forms._public_form` follows on the way out, applied here on the way in. - """ - prior = prior if isinstance(prior, dict) else {} - out = { - "id": str(prior.get("id") or raw.get("id") or ""), - "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], - "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], - "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], - "active": bool(raw.get("active", prior.get("active", True))), - # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this - # key existed must read as "approval required": the one direction a default here is - # allowed to be wrong in is the safe one, because an unattended write cannot be undone by - # revoking the bot afterwards [[default-must-pass-its-own-guard]]. - "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, - # The permission block. Same shape, same validator, same engine as a user's. - "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, - "perms_v": int(prior.get("perms_v") or 0), - "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), - "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), - } - return out - - -def _put_agent(session, agent_id, mutate): - """Read-modify-write ONE agent under the tenant's bucket, synchronously. - - `flush="sync"` because the client re-reads the list immediately after every write here, and an - async flush would let the refetch win the race and paint the pre-write state — the - [[refetch-eats-its-own-write]] shape. - """ - def _set(cur): - cur = dict(cur or {}) - rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None - nxt = mutate(rec) - if nxt is None: - cur.pop(agent_id, None) - else: - cur[agent_id] = nxt - return cur - - session.runtime.update(AGENTS_KEY, _set, flush="sync") - - -# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── -def agent_principal(agent): - """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. - - ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user - RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. - So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, - `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second - engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about - a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means - when it is true in code rather than in appearance. - - ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True - unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a - locked store. A bot has no such emergency and no hands: an agent record that could carry - `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is - not copied from the record; it is written here, every time. - - ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY - (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy - grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with - access to everything. Fail-closed from the first byte. - """ - import core.perm_scope as perm_scope - return { - "username": f"slack:{(agent or {}).get('channel') or '?'}", - "role": "agent", - "perms": (agent or {}).get("perms") or {}, - "perms_v": perm_scope.PERMS_VERSION, - # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent - # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction - # this must fail in. - } - - -def agent_may_read(agent, module): - """May this channel open `module` at all?""" - import core.perm_scope as perm_scope - if not (agent or {}).get("active", True): - return False - return bool(perm_scope.may_access(agent_principal(agent), module)) - - -def agent_visible_fields(agent, fields, module): - """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, - so a formula over a hidden column cannot leak it back through arithmetic.""" - import core.perm_scope as perm_scope - return perm_scope.visible_fields(fields, agent_principal(agent), module) - - -def agent_rows(agent, rows, module, fields, ctx=None, st=None): - """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES - rather than being ignored. - - ⭐⭐ `st` IS THE TENANT HANDLE AND IT IS HERE BECAUSE OF THE SENTENCE IN `agent_principal`: - a channel agent IS a principal of the same wall, so a permanent filter naming a - user-generated column has to mean the SAME thing here as it does on the grid (owner I16). - A door that lends the handle and a door that does not are one stored rule with two meanings. - - ⚠ PASS THE TENANT'S OWN RUNTIME — every other store read in this file takes - `session.runtime`, and a channel is not a second tenancy. It is a keyword with a `None` - default for the same reason `perm_scope.apply_row_scope`'s is: without one this is - byte-identical to the wall that shipped before the argument existed. - - ⚠ `hidden_keys` DELIBERATELY DOES NOT TAKE IT IN THIS CHANGE. Its field-grant leg - under-hides without a handle (see `perm_scope.visible_overlays`), which is a real and - separate gap on this surface — but it is the FIELD wall, it moves what a channel receives - rather than what I16 asked for, and widening two walls in one ticket is how a permission - change stops being reviewable. - """ - import core.perm_scope as perm_scope - p = agent_principal(agent) - hide = perm_scope.hidden_keys(p, module, fields) - kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st) - # Both wires, never one: the field list and the row payload are separate, and stripping only - # the first leaves the value sitting in the second where anyone can read it. - return [perm_scope.strip_row(r, hide) for r in kept] - - -# ── the credential ─────────────────────────────────────────────────────────────────────────── -def slack_creds(rt): - """This TENANT's Slack credentials from its own keychain, or None. - - ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT - HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because - `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a - tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells - out the rule: handing the environment's credential to a tenant whose admin has not stored one - is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane - SAYS so. - - ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` - (private, and in a file outside this lane's fence). Same deterministic rule: entries are - id-sorted, so "first" is stable across reads rather than dict-order luck. - """ - try: - import core.keychain as keychain - for row in keychain.list_entries(rt): - if str(row.get("type") or "") != "slack": - continue - fields = keychain.read_fields(rt, row.get("id")) - if isinstance(fields, dict) and fields.get("signing_secret"): - return fields - except Exception: # noqa: BLE001 - # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; - # the signature check below fails closed regardless, so this cannot widen anything. - return None - return None - - -# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── -def _summary(agent, modules): - """One sentence per agent for the list row — computed here, because the alternative is one - round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" - # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to - # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and - # summarised every agent as "" (no databases at all). - governed = list(modules) - perms = agent.get("perms") or {} - open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) - if not governed: - return "" - if open_n == 0: - return "No access" - restricted = sum(1 for m in governed - if (perms.get(m["key"]) or {}).get("access") - and ((perms.get(m["key"]) or {}).get("filter") - or (perms.get(m["key"]) or {}).get("hiddenFields"))) - head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) - else f"{open_n} of {len(governed)} databases") - return f"{head}, {restricted} restricted" if restricted else head - - -@router.get("/agents") -def list_channel_agents(session: Session = Depends(admin_gate)): - """This tenant's channel agents, plus whether Slack is reachable at all.""" - import routes_admin - modules = routes_admin._perm_modules(session) - rows = [] - for aid, a in sorted(_agents(session.runtime).items()): - if not isinstance(a, dict): - continue - rows.append({"id": aid, "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", - "label": a.get("label") or "", "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "summary": _summary(a, modules)}) - return {"agents": rows, "configured": slack_creds(session.runtime) is not None} - - -@router.post("/agents") -def create_channel_agent(body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - body = body if isinstance(body, dict) else {} - channel = str(body.get("channel") or "").strip() - if not channel: - raise err(400, "no_channel", "a Slack channel ID is required") - existing = _agents(session.runtime) - # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing - # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one - # place where the two answers are "may read" and "may not". - for a in existing.values(): - if isinstance(a, dict) and str(a.get("channel") or "") == channel: - raise err(409, "channel_taken", - "that channel already has an agent — edit it instead of adding a second") - aid = secrets.token_urlsafe(9) - rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, - createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) - _put_agent(session, aid, lambda _prior: rec) - fresh = _agents(session.runtime).get(aid) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell an administrator - # the agent exists when it does not. - raise err(503, "store_unavailable", "the agent was NOT created") - return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], - "label": rec["label"], "active": rec["active"], - "writeApproval": rec["writeApproval"], "summary": "No access"}} - - -@router.get("/agents/{agent_id}") -def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): - """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. - - ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and - `settings/ModulePermsList` renders it, because they are literally the same client code. A - second envelope here would mean a second parser and a second set of defaults within a wave. - """ - import routes_admin - a = _agents(session.runtime).get(str(agent_id)) - if not isinstance(a, dict): - raise err(404, "no_such_agent", "no agent with that id") - modules = routes_admin._perm_modules(session) - # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S - # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the - # key is None, so this list would have been EMPTY and the channel perms editor would have - # governed ZERO databases while looking entirely correct. A Slack channel is a principal of - # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. - governed = [m["key"] for m in modules] - stored = a.get("perms") or {} - principal = agent_principal(a) - import core.perm_scope as perm_scope - perms_out = {} - for k in governed: - e = stored.get(k) - # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled - # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a - # channel agent is not a person with a share, so a `ut_*` key it has not been granted - # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the - # table routes would give it. - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), - "filter": None, "hiddenFields": []} - return {"id": str(agent_id), "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", "label": a.get("label") or "", - "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "perms": perms_out, - # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must - # render "undeclared = denied", not the legacy-grant reading it would show at 0. - "perms_v": perm_scope.PERMS_VERSION, - # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints - # the "Everything (admin)" state for a principal that cannot have it. - "is_admin": False, - "modules": modules, - "fields_by_module": {k: routes_admin._module_fields(k, session=session) - for k in governed}} - - -@router.put("/agents/{agent_id}/perms") -def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this agent's permission block wholesale — the same validator a user's block gets.""" - import routes_admin - body = body if isinstance(body, dict) else {} - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - mods = routes_admin._perm_modules(session) - # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot - # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the - # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, - # and a second validator here is a second place for one of them to go missing. - # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal - # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left - # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives - # it ([[two-gates-can-assert-opposite-things]]). - cleaned = routes_admin._clean_perms( - body.get("perms"), - governed_keys={m["key"] for m in mods}, session=session) or {} - - import core.perm_scope as perm_scope - - def _mut(prior): - if not isinstance(prior, dict): - return None - nxt = dict(prior) - nxt["perms"] = cleaned - nxt["perms_v"] = perm_scope.PERMS_VERSION - return nxt - - _put_agent(session, str(agent_id), _mut) - fresh = _agents(session.runtime).get(str(agent_id)) or {} - if (fresh.get("perms") or {}) != cleaned: - # The one direction this must never fail in: telling an administrator the wall is up when - # it is not. - raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") - return {"id": str(agent_id), "perms": fresh.get("perms") or {}, - "perms_v": int(fresh.get("perms_v") or 0)} - - -@router.patch("/agents/{agent_id}") -def patch_channel_agent(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - body = body if isinstance(body, dict) else {} - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it - #: on the metadata route would be a second, unvalidated way to write the wall. - allowed = {k: v for k, v in body.items() - if k in ("label", "active", "writeApproval", "channelName")} - if not allowed: - raise err(400, "empty_patch", "nothing to change") - - def _mut(prior): - return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None - - _put_agent(session, str(agent_id), _mut) - import routes_admin - a = _agents(session.runtime).get(str(agent_id)) or {} - return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", "label": a.get("label") or "", - "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "summary": _summary(a, routes_admin._perm_modules(session))}} - - -@router.delete("/agents/{agent_id}") -def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - _put_agent(session, str(agent_id), lambda _prior: None) - if isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(503, "store_unavailable", "the agent was NOT removed") - return {"ok": True} - - -# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── -def _is_empty(value): - """Is this value ABSENT, as opposed to falsy? - - ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: - `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to - False is precisely the kind of write somebody needs to approve, and a naive `if not value` - drops both rows silently — the approver then reads a card that does not describe the write - they are approving. The MIT licence gate for this borrowing is recorded in - `.claude/wiki/research/opentag-adoption.md`. - """ - return value is None or (isinstance(value, str) and value.strip() == "") - - -def approval_card(action, values, agent=None): - """The card a human reads before a bot writes. `{action, rows, truncated, note}`. - - ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain - length Slack collapses a message behind "Show more", and **an approver who cannot read the - card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody - was told about is the violation, not the truncation (wave-30 R6's second sentence). - """ - rows = [{"field": str(k), "value": v} - for k, v in (values or {}).items() if not _is_empty(v)] - rows.sort(key=lambda r: r["field"]) - shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) - return { - "action": str(action or "")[:80], - "rows": shown, - "truncated": dropped, - "note": (f"{dropped} more field(s) not shown — open the record to review them before " - f"approving." if dropped else ""), - "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", - } - - -def intercept_write(agent, action, values, stash): - """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. - - ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card - naming the action and its values, and committed only on an explicit human accept. Everything - else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). - - ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` - existed requires approval. The unsafe direction here is not recoverable — revoking a bot does - not un-write what it wrote. - """ - if (agent or {}).get("writeApproval", True) is not False: - card = approval_card(action, values, agent) - stash(card) - return "await_approval", card - return "commit", None - - -def _pending(rt): - try: - found = rt.get(PENDING_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def stash_pending(rt, agent, action, values, now=None): - """Store one awaiting-approval mutation and return its token. - - The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the - action's id: a guessable value on an unauthenticated door is a way to approve somebody else's - write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. - """ - token = secrets.token_urlsafe(18) - row = {"at": float(now if now is not None else time.time()), - "channel": str((agent or {}).get("channel") or ""), - "action": str(action or "")[:80], - "values": values if isinstance(values, dict) else {}} - rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") - return token - - -#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. -#: -#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE -#: place an approved write may be performed, so it is the place a negative control can bite — but -#: a gate is only as honest as what it guards. Until something registers here, an approved card -#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a -#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. -_COMMITTERS = {} - - -def register_committer(kind, fn): - """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" - _COMMITTERS[str(kind)] = fn - - -def commit_pending(rt, token, approved, now=None): - """Resolve ONE approval card. Returns one of - `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. - - ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's - `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the - route could not bite while the route had no write in it at all — it would stay green with the - guard deleted, because there would be nothing for the guard to be protecting anything from - ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's - first draft). Putting the commit behind ONE named seam gives the control a real subject: the - gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it - exactly once. - - ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that - survives its own click is a replayable write, and "approve twice" must never mean "write - twice". The delete is `flush="sync"` for the same reason. - """ - now = float(now if now is not None else time.time()) - row = _pending(rt).get(str(token)) - if not isinstance(row, dict): - return "unknown" - # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. - rt.update(PENDING_KEY, - lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, - flush="sync") - if now - float(row.get("at") or 0) > APPROVAL_TTL_S: - return "expired" - if not approved: - # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. - return "rejected" - fn = _COMMITTERS.get(str(row.get("action") or "")) - if fn is None: - return "no_committer" - try: - fn(rt, row) - except Exception: # noqa: BLE001 - # A committer that raises must not read as a commit. The card is already consumed, so the - # honest report is that it failed, and the human asks again. - return "failed" - return "committed" - - -# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── -#: Sliding window, keyed on the SLACK TEAM, never the client IP. -#: -#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own -#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one -#: busy tenant would spend the allowance for all of them, and the symptom would be another -#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. -#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER -#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature -#: check is a single HMAC). -RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 -_HITS: dict = {} - - -def _rate_ok(key, now): - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - seen.append(now) - _HITS[key] = seen - if len(_HITS) > 4096: - for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: - _HITS.pop(k, None) - return len(seen) <= RATE_PER_WINDOW - - -async def _raw_body(request): - """The body as RAW BYTES, capped. - - ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string - `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is - otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim - copy of it could not verify a Slack signature at all: re-serialising the dict changes - whitespace and key order and the HMAC no longer matches. The bug would look like "Slack - signatures are always invalid", which is indistinguishable from a wrong secret. - - ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered - the whole body before the handler's first line, so a content-length check inside the handler - caps nothing (`routes_forms`' own scar, recorded in its docstring). - """ - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > MAX_BODY_BYTES: - raise err(413, "body_too_large", "that request is too large") - chunks.append(chunk) - return b"".join(chunks) - - -def _refuse(): - """ONE refusal for every reason. No oracle. - - A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel - with no agent all answer identically — otherwise the door tells an unauthenticated caller - which tenants exist and which channels are configured, which is the question it was built to - not answer. `routes_forms._refuse` takes the same posture for the same reason. - """ - return err(403, "bad_request", "that request could not be verified") - - -def _verify(secret, timestamp, raw, signature, now): - """Slack's v0 signature, with the replay window checked FIRST.""" - try: - ts = int(str(timestamp or "0")) - except ValueError: - return False - if abs(now - ts) > SLACK_MAX_SKEW_S: - return False - base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") - import hashlib - expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() - return hmac.compare_digest(expected, str(signature or "")) - - -def _resolve_tenant(raw, timestamp, signature, now): - """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. - - ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` - would let an unauthenticated caller name the tenant it wants to be — the exact widening - `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured - tenant's secret is tried and the one that VERIFIES names the tenant. - - ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not - depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the - same reason — a public door has no session to ask. - """ - from harness import runtime as _rt - hit = None - try: - slugs = list(_rt.known_tenants()) - except Exception: # noqa: BLE001 - return None, None, None - for slug in slugs: - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - creds = slack_creds(rt) - if not creds: - continue - if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: - hit = (slug, rt, creds) - return hit if hit else (None, None, None) - - -@router.post("/slack/events") -async def slack_events(request: Request): - """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. - - ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, - no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves - this deployment (amendment A1). - """ - now = time.time() - raw = await _raw_body(request) - ts = request.headers.get("x-slack-request-timestamp") - sig = request.headers.get("x-slack-signature") - slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) - if not rt: - raise _refuse() - - try: - payload = json.loads(raw or b"{}") - except ValueError: - raise _refuse() - if not isinstance(payload, dict): - raise _refuse() - - # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an - # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. - if payload.get("type") == "url_verification": - return {"challenge": str(payload.get("challenge") or "")[:512]} - - team = str(payload.get("team_id") or slug) - if not _rate_ok(f"{slug}:{team}", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - - event = payload.get("event") if isinstance(payload.get("event"), dict) else {} - channel = str(event.get("channel") or "") - agent = None - for a in _agents(rt).values(): - if isinstance(a, dict) and str(a.get("channel") or "") == channel: - agent = a - break - # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries - # a non-2xx up to three times, so refusing here would turn "this channel is not configured" - # into three refusals per message, and the retry storm would be the only visible symptom. - # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which - # channels this workspace has configured. - if not isinstance(agent, dict) or agent.get("active", True) is False: - return {"ok": True} - - # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the - # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a - # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing - # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built - # answerer that returns something plausible is worse than one that returns nothing. - # Booked rather than faked [[flag-shipped-without-its-writer]]. - return {"ok": True} - - -@router.post("/slack/interact") -async def slack_interact(request: Request): - """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" - now = time.time() - raw = await _raw_body(request) - slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), - request.headers.get("x-slack-signature"), now) - if not rt: - raise _refuse() - # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. - from urllib.parse import parse_qs - try: - form = parse_qs(raw.decode("utf-8")) - payload = json.loads((form.get("payload") or ["{}"])[0]) - except Exception: # noqa: BLE001 - raise _refuse() - if not isinstance(payload, dict): - raise _refuse() - if not _rate_ok(f"{slug}:interact", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - - actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] - choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) - token = str(choice.get("value") or "") - # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, - # truncated or unknown action id: the direction that can be wrong here without anyone - # noticing is the one that writes. - approved = str(choice.get("action_id") or "") == "approve" - # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative - # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all - # live there; this handler only turns its verdict into a sentence. - verdict = commit_pending(rt, token, approved, now=now) - return {"text": { - # An unknown, already-used or expired token is not an error: a person clicking a stale - # card should be told it is stale, not shown a failure. - "unknown": "That approval is no longer available. Ask the bot again.", - "expired": "That approval expired before it was answered. Ask the bot again.", - "rejected": "Rejected. Nothing was changed.", - # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, - # so an approved card today performs nothing and says exactly that rather than "Approved." - "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", - "failed": "That change could not be completed. Nothing was saved; ask the bot again.", - "committed": "Approved.", - }.get(verdict, "That approval is no longer available. Ask the bot again.")} - - -@router.get("/slack/health") -def slack_health(session: Session = Depends(require_session)): - """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — - enumerating the others would answer a question about our customer list.""" - creds = slack_creds(session.runtime) - return {"configured": creds is not None, - "agents": len(_agents(session.runtime)), - "hasBotToken": bool((creds or {}).get("bot_token"))} +"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a +person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) + +Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel +whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle +permissioning per user."* + +──────────────────────────────────────────────────────────────────────────────────────────────── +⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). + +`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since +2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is +`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own +`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its +authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a +live specimen of identity that LOOKS like a permission and is not. + +`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per +principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` +below, which is the whole projection and is nine lines. Study: +`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). + +What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` +below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a +second Node process, a persistent outbound socket and each tenant's Slack credentials held by a +third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven +unauthenticated-public shape plus Slack's signing-secret check. + +──────────────────────────────────────────────────────────────────────────────────────────────── +⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every +human in that channel reads through the same rules. That is the honest reading of "a bot per +channel" and the pane says it out loud, because the alternative is an admin assuming per-person +scoping from a screen that looks exactly like the per-person one. + +⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. +A wall bound to `#sales` silently re-points the day somebody renames the channel. +""" +import hmac +import json +import os +import secrets +import time +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends, Request + +from deps import Session, err, require_session +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. +AGENTS_KEY = "slack_agents" + +#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. +PENDING_KEY = "slack_pending_writes" + +#: How long an approval card stays clickable. An approval is a statement about the world as it was +#: when the card was rendered; an hour later the values it names may no longer be the values it +#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. +APPROVAL_TTL_S = 15 * 60 + +#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. +SLACK_MAX_SKEW_S = 60 * 5 + +MAX_BODY_BYTES = 64 * 1024 + +#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied +#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: +#: past a certain length Slack collapses a message behind "Show more", and a human clicking +#: Approve on a card whose tail they never saw is worse than no card at all. +APPROVAL_MAX_ROWS = 12 + + +# ── the tenant's agent records ─────────────────────────────────────────────────────────────── +def _agents(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this tenant has no agents", never to a 500 on the settings pane.""" + try: + found = rt.get(AGENTS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _clean_agent(raw, prior=None): + """One stored record, built KEY BY KEY. + + ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; + a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule + `routes_forms._public_form` follows on the way out, applied here on the way in. + """ + prior = prior if isinstance(prior, dict) else {} + out = { + "id": str(prior.get("id") or raw.get("id") or ""), + "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], + "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], + "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], + "active": bool(raw.get("active", prior.get("active", True))), + # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this + # key existed must read as "approval required": the one direction a default here is + # allowed to be wrong in is the safe one, because an unattended write cannot be undone by + # revoking the bot afterwards [[default-must-pass-its-own-guard]]. + "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, + # The permission block. Same shape, same validator, same engine as a user's. + "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, + "perms_v": int(prior.get("perms_v") or 0), + "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), + "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), + } + return out + + +def _put_agent(session, agent_id, mutate): + """Read-modify-write ONE agent under the tenant's bucket, synchronously. + + `flush="sync"` because the client re-reads the list immediately after every write here, and an + async flush would let the refetch win the race and paint the pre-write state — the + [[refetch-eats-its-own-write]] shape. + """ + def _set(cur): + cur = dict(cur or {}) + rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None + nxt = mutate(rec) + if nxt is None: + cur.pop(agent_id, None) + else: + cur[agent_id] = nxt + return cur + + session.runtime.update(AGENTS_KEY, _set, flush="sync") + + +# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── +def agent_principal(agent): + """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. + + ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user + RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. + So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, + `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second + engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about + a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means + when it is true in code rather than in appearance. + + ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True + unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a + locked store. A bot has no such emergency and no hands: an agent record that could carry + `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is + not copied from the record; it is written here, every time. + + ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY + (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy + grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with + access to everything. Fail-closed from the first byte. + """ + import core.perm_scope as perm_scope + return { + "username": f"slack:{(agent or {}).get('channel') or '?'}", + "role": "agent", + "perms": (agent or {}).get("perms") or {}, + "perms_v": perm_scope.PERMS_VERSION, + # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent + # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction + # this must fail in. + } + + +def agent_may_read(agent, module): + """May this channel open `module` at all?""" + import core.perm_scope as perm_scope + if not (agent or {}).get("active", True): + return False + return bool(perm_scope.may_access(agent_principal(agent), module)) + + +def agent_visible_fields(agent, fields, module): + """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, + so a formula over a hidden column cannot leak it back through arithmetic.""" + import core.perm_scope as perm_scope + return perm_scope.visible_fields(fields, agent_principal(agent), module) + + +def agent_rows(agent, rows, module, fields, ctx=None, st=None): + """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES + rather than being ignored. + + ⭐⭐ `st` IS THE TENANT HANDLE AND IT IS HERE BECAUSE OF THE SENTENCE IN `agent_principal`: + a channel agent IS a principal of the same wall, so a permanent filter naming a + user-generated column has to mean the SAME thing here as it does on the grid (owner I16). + A door that lends the handle and a door that does not are one stored rule with two meanings. + + ⚠ PASS THE TENANT'S OWN RUNTIME — every other store read in this file takes + `session.runtime`, and a channel is not a second tenancy. It is a keyword with a `None` + default for the same reason `perm_scope.apply_row_scope`'s is: without one this is + byte-identical to the wall that shipped before the argument existed. + + ⚠ `hidden_keys` DELIBERATELY DOES NOT TAKE IT IN THIS CHANGE. Its field-grant leg + under-hides without a handle (see `perm_scope.visible_overlays`), which is a real and + separate gap on this surface — but it is the FIELD wall, it moves what a channel receives + rather than what I16 asked for, and widening two walls in one ticket is how a permission + change stops being reviewable. + """ + import core.perm_scope as perm_scope + p = agent_principal(agent) + hide = perm_scope.hidden_keys(p, module, fields) + kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st) + # Both wires, never one: the field list and the row payload are separate, and stripping only + # the first leaves the value sitting in the second where anyone can read it. + return [perm_scope.strip_row(r, hide) for r in kept] + + +# ── the credential ─────────────────────────────────────────────────────────────────────────── +def slack_creds(rt): + """This TENANT's Slack credentials from its own keychain, or None. + + ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT + HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because + `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a + tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells + out the rule: handing the environment's credential to a tenant whose admin has not stored one + is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane + SAYS so. + + ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` + (private, and in a file outside this lane's fence). Same deterministic rule: entries are + id-sorted, so "first" is stable across reads rather than dict-order luck. + """ + try: + import core.keychain as keychain + for row in keychain.list_entries(rt): + if str(row.get("type") or "") != "slack": + continue + fields = keychain.read_fields(rt, row.get("id")) + if isinstance(fields, dict) and fields.get("signing_secret"): + return fields + except Exception: # noqa: BLE001 + # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; + # the signature check below fails closed regardless, so this cannot widen anything. + return None + return None + + +# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── +def _summary(agent, modules): + """One sentence per agent for the list row — computed here, because the alternative is one + round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" + # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to + # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and + # summarised every agent as "" (no databases at all). + governed = list(modules) + perms = agent.get("perms") or {} + open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) + if not governed: + return "" + if open_n == 0: + return "No access" + restricted = sum(1 for m in governed + if (perms.get(m["key"]) or {}).get("access") + and ((perms.get(m["key"]) or {}).get("filter") + or (perms.get(m["key"]) or {}).get("hiddenFields"))) + head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) + else f"{open_n} of {len(governed)} databases") + return f"{head}, {restricted} restricted" if restricted else head + + +@router.get("/agents") +def list_channel_agents(session: Session = Depends(admin_gate)): + """This tenant's channel agents, plus whether Slack is reachable at all.""" + import routes_admin + modules = routes_admin._perm_modules(session) + rows = [] + for aid, a in sorted(_agents(session.runtime).items()): + if not isinstance(a, dict): + continue + rows.append({"id": aid, "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", + "label": a.get("label") or "", "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, modules)}) + return {"agents": rows, "configured": slack_creds(session.runtime) is not None} + + +@router.post("/agents") +def create_channel_agent(body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + channel = str(body.get("channel") or "").strip() + if not channel: + raise err(400, "no_channel", "a Slack channel ID is required") + existing = _agents(session.runtime) + # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing + # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one + # place where the two answers are "may read" and "may not". + for a in existing.values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + raise err(409, "channel_taken", + "that channel already has an agent — edit it instead of adding a second") + aid = secrets.token_urlsafe(9) + rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, + createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) + _put_agent(session, aid, lambda _prior: rec) + fresh = _agents(session.runtime).get(aid) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell an administrator + # the agent exists when it does not. + raise err(503, "store_unavailable", "the agent was NOT created") + return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], + "label": rec["label"], "active": rec["active"], + "writeApproval": rec["writeApproval"], "summary": "No access"}} + + +@router.get("/agents/{agent_id}") +def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. + + ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and + `settings/ModulePermsList` renders it, because they are literally the same client code. A + second envelope here would mean a second parser and a second set of defaults within a wave. + """ + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) + if not isinstance(a, dict): + raise err(404, "no_such_agent", "no agent with that id") + modules = routes_admin._perm_modules(session) + # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S + # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the + # key is None, so this list would have been EMPTY and the channel perms editor would have + # governed ZERO databases while looking entirely correct. A Slack channel is a principal of + # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. + governed = [m["key"] for m in modules] + stored = a.get("perms") or {} + principal = agent_principal(a) + import core.perm_scope as perm_scope + perms_out = {} + for k in governed: + e = stored.get(k) + # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled + # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a + # channel agent is not a person with a share, so a `ut_*` key it has not been granted + # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the + # table routes would give it. + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), + "filter": None, "hiddenFields": []} + return {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "perms": perms_out, + # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must + # render "undeclared = denied", not the legacy-grant reading it would show at 0. + "perms_v": perm_scope.PERMS_VERSION, + # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints + # the "Everything (admin)" state for a principal that cannot have it. + "is_admin": False, + "modules": modules, + "fields_by_module": {k: routes_admin._module_fields(k, session=session) + for k in governed}} + + +@router.put("/agents/{agent_id}/perms") +def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this agent's permission block wholesale — the same validator a user's block gets.""" + import routes_admin + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + mods = routes_admin._perm_modules(session) + # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot + # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the + # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, + # and a second validator here is a second place for one of them to go missing. + # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal + # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left + # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives + # it ([[two-gates-can-assert-opposite-things]]). + cleaned = routes_admin._clean_perms( + body.get("perms"), + governed_keys={m["key"] for m in mods}, session=session) or {} + + import core.perm_scope as perm_scope + + def _mut(prior): + if not isinstance(prior, dict): + return None + nxt = dict(prior) + nxt["perms"] = cleaned + nxt["perms_v"] = perm_scope.PERMS_VERSION + return nxt + + _put_agent(session, str(agent_id), _mut) + fresh = _agents(session.runtime).get(str(agent_id)) or {} + if (fresh.get("perms") or {}) != cleaned: + # The one direction this must never fail in: telling an administrator the wall is up when + # it is not. + raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") + return {"id": str(agent_id), "perms": fresh.get("perms") or {}, + "perms_v": int(fresh.get("perms_v") or 0)} + + +@router.patch("/agents/{agent_id}") +def patch_channel_agent(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it + #: on the metadata route would be a second, unvalidated way to write the wall. + allowed = {k: v for k, v in body.items() + if k in ("label", "active", "writeApproval", "channelName")} + if not allowed: + raise err(400, "empty_patch", "nothing to change") + + def _mut(prior): + return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None + + _put_agent(session, str(agent_id), _mut) + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) or {} + return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, routes_admin._perm_modules(session))}} + + +@router.delete("/agents/{agent_id}") +def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + _put_agent(session, str(agent_id), lambda _prior: None) + if isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(503, "store_unavailable", "the agent was NOT removed") + return {"ok": True} + + +# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── +def _is_empty(value): + """Is this value ABSENT, as opposed to falsy? + + ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: + `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to + False is precisely the kind of write somebody needs to approve, and a naive `if not value` + drops both rows silently — the approver then reads a card that does not describe the write + they are approving. The MIT licence gate for this borrowing is recorded in + `.claude/wiki/research/opentag-adoption.md`. + """ + return value is None or (isinstance(value, str) and value.strip() == "") + + +def approval_card(action, values, agent=None): + """The card a human reads before a bot writes. `{action, rows, truncated, note}`. + + ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain + length Slack collapses a message behind "Show more", and **an approver who cannot read the + card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody + was told about is the violation, not the truncation (wave-30 R6's second sentence). + """ + rows = [{"field": str(k), "value": v} + for k, v in (values or {}).items() if not _is_empty(v)] + rows.sort(key=lambda r: r["field"]) + shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) + return { + "action": str(action or "")[:80], + "rows": shown, + "truncated": dropped, + "note": (f"{dropped} more field(s) not shown — open the record to review them before " + f"approving." if dropped else ""), + "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", + } + + +def intercept_write(agent, action, values, stash): + """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. + + ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card + naming the action and its values, and committed only on an explicit human accept. Everything + else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). + + ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` + existed requires approval. The unsafe direction here is not recoverable — revoking a bot does + not un-write what it wrote. + """ + if (agent or {}).get("writeApproval", True) is not False: + card = approval_card(action, values, agent) + stash(card) + return "await_approval", card + return "commit", None + + +def _pending(rt): + try: + found = rt.get(PENDING_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def stash_pending(rt, agent, action, values, now=None): + """Store one awaiting-approval mutation and return its token. + + The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the + action's id: a guessable value on an unauthenticated door is a way to approve somebody else's + write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. + """ + token = secrets.token_urlsafe(18) + row = {"at": float(now if now is not None else time.time()), + "channel": str((agent or {}).get("channel") or ""), + "action": str(action or "")[:80], + "values": values if isinstance(values, dict) else {}} + rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") + return token + + +#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. +#: +#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE +#: place an approved write may be performed, so it is the place a negative control can bite — but +#: a gate is only as honest as what it guards. Until something registers here, an approved card +#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a +#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. +_COMMITTERS = {} + + +def register_committer(kind, fn): + """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" + _COMMITTERS[str(kind)] = fn + + +def commit_pending(rt, token, approved, now=None): + """Resolve ONE approval card. Returns one of + `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. + + ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's + `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the + route could not bite while the route had no write in it at all — it would stay green with the + guard deleted, because there would be nothing for the guard to be protecting anything from + ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's + first draft). Putting the commit behind ONE named seam gives the control a real subject: the + gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it + exactly once. + + ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that + survives its own click is a replayable write, and "approve twice" must never mean "write + twice". The delete is `flush="sync"` for the same reason. + """ + now = float(now if now is not None else time.time()) + row = _pending(rt).get(str(token)) + if not isinstance(row, dict): + return "unknown" + # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. + rt.update(PENDING_KEY, + lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, + flush="sync") + if now - float(row.get("at") or 0) > APPROVAL_TTL_S: + return "expired" + if not approved: + # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. + return "rejected" + fn = _COMMITTERS.get(str(row.get("action") or "")) + if fn is None: + return "no_committer" + try: + fn(rt, row) + except Exception: # noqa: BLE001 + # A committer that raises must not read as a commit. The card is already consumed, so the + # honest report is that it failed, and the human asks again. + return "failed" + return "committed" + + +# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── +#: Sliding window, keyed on the SLACK TEAM, never the client IP. +#: +#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own +#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one +#: busy tenant would spend the allowance for all of them, and the symptom would be another +#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. +#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER +#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature +#: check is a single HMAC). +RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 +_HITS: dict = {} + + +def _rate_ok(key, now): + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + return len(seen) <= RATE_PER_WINDOW + + +async def _raw_body(request): + """The body as RAW BYTES, capped. + + ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string + `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is + otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim + copy of it could not verify a Slack signature at all: re-serialising the dict changes + whitespace and key order and the HMAC no longer matches. The bug would look like "Slack + signatures are always invalid", which is indistinguishable from a wrong secret. + + ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered + the whole body before the handler's first line, so a content-length check inside the handler + caps nothing (`routes_forms`' own scar, recorded in its docstring). + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + return b"".join(chunks) + + +def _refuse(): + """ONE refusal for every reason. No oracle. + + A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel + with no agent all answer identically — otherwise the door tells an unauthenticated caller + which tenants exist and which channels are configured, which is the question it was built to + not answer. `routes_forms._refuse` takes the same posture for the same reason. + """ + return err(403, "bad_request", "that request could not be verified") + + +def _verify(secret, timestamp, raw, signature, now): + """Slack's v0 signature, with the replay window checked FIRST.""" + try: + ts = int(str(timestamp or "0")) + except ValueError: + return False + if abs(now - ts) > SLACK_MAX_SKEW_S: + return False + base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") + import hashlib + expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, str(signature or "")) + + +def _resolve_tenant(raw, timestamp, signature, now): + """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. + + ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` + would let an unauthenticated caller name the tenant it wants to be — the exact widening + `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured + tenant's secret is tried and the one that VERIFIES names the tenant. + + ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not + depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the + same reason — a public door has no session to ask. + """ + from harness import runtime as _rt + hit = None + try: + slugs = list(_rt.known_tenants()) + except Exception: # noqa: BLE001 + return None, None, None + for slug in slugs: + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + creds = slack_creds(rt) + if not creds: + continue + if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: + hit = (slug, rt, creds) + return hit if hit else (None, None, None) + + +@router.post("/slack/events") +async def slack_events(request: Request): + """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. + + ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, + no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves + this deployment (amendment A1). + """ + now = time.time() + raw = await _raw_body(request) + ts = request.headers.get("x-slack-request-timestamp") + sig = request.headers.get("x-slack-signature") + slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) + if not rt: + raise _refuse() + + try: + payload = json.loads(raw or b"{}") + except ValueError: + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + + # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an + # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. + if payload.get("type") == "url_verification": + return {"challenge": str(payload.get("challenge") or "")[:512]} + + team = str(payload.get("team_id") or slug) + if not _rate_ok(f"{slug}:{team}", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + event = payload.get("event") if isinstance(payload.get("event"), dict) else {} + channel = str(event.get("channel") or "") + agent = None + for a in _agents(rt).values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + agent = a + break + # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries + # a non-2xx up to three times, so refusing here would turn "this channel is not configured" + # into three refusals per message, and the retry storm would be the only visible symptom. + # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which + # channels this workspace has configured. + if not isinstance(agent, dict) or agent.get("active", True) is False: + return {"ok": True} + + # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the + # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a + # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing + # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built + # answerer that returns something plausible is worse than one that returns nothing. + # Booked rather than faked [[flag-shipped-without-its-writer]]. + return {"ok": True} + + +@router.post("/slack/interact") +async def slack_interact(request: Request): + """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" + now = time.time() + raw = await _raw_body(request) + slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), + request.headers.get("x-slack-signature"), now) + if not rt: + raise _refuse() + # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. + from urllib.parse import parse_qs + try: + form = parse_qs(raw.decode("utf-8")) + payload = json.loads((form.get("payload") or ["{}"])[0]) + except Exception: # noqa: BLE001 + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + if not _rate_ok(f"{slug}:interact", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] + choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) + token = str(choice.get("value") or "") + # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, + # truncated or unknown action id: the direction that can be wrong here without anyone + # noticing is the one that writes. + approved = str(choice.get("action_id") or "") == "approve" + # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative + # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all + # live there; this handler only turns its verdict into a sentence. + verdict = commit_pending(rt, token, approved, now=now) + return {"text": { + # An unknown, already-used or expired token is not an error: a person clicking a stale + # card should be told it is stale, not shown a failure. + "unknown": "That approval is no longer available. Ask the bot again.", + "expired": "That approval expired before it was answered. Ask the bot again.", + "rejected": "Rejected. Nothing was changed.", + # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, + # so an approved card today performs nothing and says exactly that rather than "Approved." + "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", + "failed": "That change could not be completed. Nothing was saved; ask the bot again.", + "committed": "Approved.", + }.get(verdict, "That approval is no longer available. Ask the bot again.")} + + +@router.get("/slack/health") +def slack_health(session: Session = Depends(require_session)): + """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — + enumerating the others would answer a question about our customer list.""" + creds = slack_creds(session.runtime) + return {"configured": creds is not None, + "agents": len(_agents(session.runtime)), + "hasBotToken": bool((creds or {}).get("bot_token"))} diff --git a/api/routes_statements.py b/api/routes_statements.py index c39730724d527120c094a5afc94652703bb1de87..c2cbedf8619c07e441bc5a6f1f80ffb5982b0772 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -1,298 +1,298 @@ -"""routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6). - -⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is -read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists -exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and -add no write of their own. - -WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the -Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from -the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so -when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than -dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same -preview, same test-send, same two-step confirm. - -THE THREE GUARDRAILS, and where each is enforced: - 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route, - payload or UI can bypass it. These routes only REPORT it; they never re-implement the check. - 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`. - 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()` - reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to - TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab - admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the - single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall - rather than copy one. - -NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no -"send all" parameter, deliberately — the confirm step is a product requirement, not a formality. -""" -from fastapi import APIRouter, Body, Depends - -from deps import Session, err -from routes_admin import admin_gate - -# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this -# way round. `main.py` already imports both modules, so this adds no load. -import automation_engine as engine - -router = APIRouter(prefix="/api/v1") - -#: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the -#: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read. -_TTL = 1800 -_cache = {"at": 0.0, "rows": None} - - -def _cs(): - import modules.collections_send as cs - return cs - - -def _royal_only(session: Session) -> Session: - """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is - tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. - - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. - - ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an - agent step this wave, so the automation engine has to answer the same question at two more - doors (which tenants see the action, which tenants may store it). C8's words are "the predicate - is imported, never re-expressed", and this is the direction that import can run: the engine - imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and - `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. - ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the - surface is invisible rather than forbidden. The engine must not know about status codes.""" - if not engine.is_statement_tenant(session.runtime): - raise err(404, "not_found", "statements are not configured for this workspace") - return session - - -def _gate(session: Session = Depends(admin_gate)) -> Session: - return _royal_only(session) - - -def _rows(force=False): - import time - cs = _cs() - if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL: - _cache["rows"] = cs.load_collection_list(cs.Odoo()) - _cache["at"] = time.time() - return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"])) - - -def _public(row): - """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because - the client needs a stable row identity that is not the display name).""" - return {k: v for k, v in row.items() if not str(k).startswith("_")} - - -@router.get("/admin/statements") -def statements(refresh: int = 0, session: Session = Depends(_gate)): - """The worklist + everything the sender UI needs to render itself honestly.""" - cs = _cs() - try: - rows, loaded_at = _rows(force=bool(refresh)) - except Exception as e: - raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}") - return { - "rows": [_public(r) for r in rows], - "loadedAt": loaded_at, - # The guardrail is reported, never decided, here — the data layer owns it. - "safeMode": bool(cs.SAFE_MODE), - "safeRecipients": sorted(cs.SAFE_RECIPIENTS), - "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, - "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, - "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, - "footer": cs.DEFAULT_FOOTER}, - # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements - # agent's tier filter cannot come to mean different things. - "tiers": list(engine.STATEMENT_TIERS), - } - - -def _find(rows, customer): - return next((r for r in rows if r.get("Customer") == customer), None) - - -@router.post("/admin/statements/preview") -def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): - """Render ONE customer's statement exactly as the send path would.""" - cs = _cs() - body = body or {} - rows, _ = _rows() - row = _find(rows, body.get("customer")) - if row is None: - raise err(404, "not_found", "no such customer on the collection list") - t = body.get("templates") or {} - import datetime as dt - month = dt.date.today().strftime("%B %Y") - subject = (t.get("subject") or cs.DEFAULT_SUBJECT) - try: - subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) - except Exception: # noqa: BLE001 - # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they - # can see what they typed rather than getting an opaque error. - # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. - # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a - # TypeError, and every one of them is the same user mistake this branch was written for. - # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the - # one screen somebody opens to avoid mailing 200 people the wrong thing. - pass - return { - "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, - t.get("footer") or cs.DEFAULT_FOOTER), - "to": row.get("Email") or "", - "subject": subject, - } - - -@router.post("/admin/statements/send") -def send(body: dict = Body(default=None), session: Session = Depends(_gate)): - """Queue statements. Returns per-customer outcomes — NEVER a bare count. - - `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the - data layer refuses an address outside the allow-list), which is why the route does not check it. - """ - cs = _cs() - body = body or {} - names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] - if not names: - raise err(400, "bad_request", "name at least one customer") - # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A - # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent - # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) - # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On - # the one route in this product that is allowed to write to Odoo, the difference between a - # rehearsal and mailing a live customer was one empty string. - # - # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` - # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature - # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key - # being present, or `test: true`, is that statement, and an empty address beside it is the - # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` - # asserted with no key at all. - # - # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. - # `automationApi.testSendStatement` refuses an empty address before the request is built, and - # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client - # wall is bypassable by construction (anything can POST); this one is not. - declared_test = ("overrideTo" in body) or bool(body.get("test")) - override = str(body.get("overrideTo") or "").strip() - if declared_test and not override: - raise err(400, "no_override_address", - "a test send needs the address to send the test to. Without one this would " - "mail the customer's own address, which is the opposite of a test") - override = override or None - if override and len(names) != 1: - raise err(400, "bad_request", "a test send takes exactly one customer") - t = body.get("templates") or {} - rows, _ = _rows() - - sent, failed, skipped = [], [], [] - for name in names: - row = _find(rows, name) - if row is None: - failed.append({"customer": name, "error": "not on the current collection list"}) - continue - if not override and not row.get("Email"): - # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures - # keeps "we could not" distinct from "there was nowhere to send". - skipped.append({"customer": name, "reason": "no email address on the customer record"}) - continue - try: - mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT, - t.get("intro") or cs.DEFAULT_INTRO, - t.get("footer") or cs.DEFAULT_FOOTER, override_to=override) - sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid}) - except Exception as e: - # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal - # and an Odoo error are both "this one did not go", each with its own honest message. - failed.append({"customer": name, "error": str(e)[:200]}) - return {"sent": sent, "failed": failed, "skipped": skipped, - "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} - - -# --------------------------------------------------------------------------------------------- -# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. -# -# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch -# and parks it on the automation definition, and it imports no mail path at all. THIS is the only -# door that releases one, and it is deliberately built out of the parts already here: -# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three -# endpoints use. Not a copy of the checks: the same object. -# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal -# happens in the DATA LAYER, where no route, payload or UI can reach around it. -# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. -# -# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked -# `html` is what a person APPROVED and is what the review screen shows; but a balance can move -# between the parking and the click, and mailing a figure we know to be stale is worse than mailing -# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the -# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely -# (they paid) is reported as skipped rather than invoiced. - -@router.get("/admin/statements/agent/{auto_id}") -def agent_batch(auto_id: str, session: Session = Depends(_gate)): - """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - pending = defn.get("pendingStatements") - if not isinstance(pending, dict): - return {"pending": None} - # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is - # megabytes, and the review screen needs the count and the names to decide. `preview` already - # renders ONE on demand. - return {"pending": { - "ts": pending.get("ts"), "count": int(pending.get("count") or 0), - "notes": list(pending.get("notes") or []), - "items": [{k: v for k, v in it.items() if k != "html"} - for it in (pending.get("items") or [])], - "safeMode": bool(_cs().SAFE_MODE)}} - - -@router.post("/admin/statements/agent/{auto_id}/send") -def agent_send(auto_id: str, body: dict = Body(default=None), - session: Session = Depends(_gate)): - """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. - - ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send - that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) - arriving on the one route in this system that talks to a mail server. - """ - cs = _cs() - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - pending = defn.get("pendingStatements") - items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] - if not items: - raise err(404, "no_batch", "there is nothing parked for this agent to send") - # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it - # may only NARROW the parked batch. A name that was never parked cannot be introduced by the - # payload, or the review stops being what authorised the send. - want = {str(n) for n in (body or {}).get("customers") or []} - if want: - items = [it for it in items if str(it.get("customer")) in want] - if not items: - raise err(400, "bad_request", "none of those customers are in the parked batch") - rows, _ = _rows() - sent, failed, skipped = [], [], [] - for it in items: - name = str(it.get("customer") or "") - row = _find(rows, name) - if row is None: - skipped.append({"customer": name, - "reason": "no longer on the collection list, so nothing is owed"}) - continue - try: - # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE - # `queue_statement`, so the guardrail cannot be argued with from here. - mid = cs.queue_statement(cs.Odoo(), row, - str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, - cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) - sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) - except Exception as e: # noqa: BLE001 - failed.append({"customer": name, "error": str(e)[:200]}) - # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would - # lose the list of who still needs a statement, and nobody would know to look. - if sent and not failed: - engine.clear_statements(session.runtime, auto_id) - return {"sent": sent, "failed": failed, "skipped": skipped, - "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} +"""routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6). + +⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is +read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists +exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and +add no write of their own. + +WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the +Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from +the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so +when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than +dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same +preview, same test-send, same two-step confirm. + +THE THREE GUARDRAILS, and where each is enforced: + 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route, + payload or UI can bypass it. These routes only REPORT it; they never re-implement the check. + 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`. + 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()` + reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to + TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab + admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the + single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall + rather than copy one. + +NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no +"send all" parameter, deliberately — the confirm step is a product requirement, not a formality. +""" +from fastapi import APIRouter, Body, Depends + +from deps import Session, err +from routes_admin import admin_gate + +# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this +# way round. `main.py` already imports both modules, so this adds no load. +import automation_engine as engine + +router = APIRouter(prefix="/api/v1") + +#: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the +#: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read. +_TTL = 1800 +_cache = {"at": 0.0, "rows": None} + + +def _cs(): + import modules.collections_send as cs + return cs + + +def _royal_only(session: Session) -> Session: + """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is + tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. + + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. + + ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an + agent step this wave, so the automation engine has to answer the same question at two more + doors (which tenants see the action, which tenants may store it). C8's words are "the predicate + is imported, never re-expressed", and this is the direction that import can run: the engine + imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and + `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. + ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the + surface is invisible rather than forbidden. The engine must not know about status codes.""" + if not engine.is_statement_tenant(session.runtime): + raise err(404, "not_found", "statements are not configured for this workspace") + return session + + +def _gate(session: Session = Depends(admin_gate)) -> Session: + return _royal_only(session) + + +def _rows(force=False): + import time + cs = _cs() + if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL: + _cache["rows"] = cs.load_collection_list(cs.Odoo()) + _cache["at"] = time.time() + return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"])) + + +def _public(row): + """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because + the client needs a stable row identity that is not the display name).""" + return {k: v for k, v in row.items() if not str(k).startswith("_")} + + +@router.get("/admin/statements") +def statements(refresh: int = 0, session: Session = Depends(_gate)): + """The worklist + everything the sender UI needs to render itself honestly.""" + cs = _cs() + try: + rows, loaded_at = _rows(force=bool(refresh)) + except Exception as e: + raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}") + return { + "rows": [_public(r) for r in rows], + "loadedAt": loaded_at, + # The guardrail is reported, never decided, here — the data layer owns it. + "safeMode": bool(cs.SAFE_MODE), + "safeRecipients": sorted(cs.SAFE_RECIPIENTS), + "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, + "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, + "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, + "footer": cs.DEFAULT_FOOTER}, + # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements + # agent's tier filter cannot come to mean different things. + "tiers": list(engine.STATEMENT_TIERS), + } + + +def _find(rows, customer): + return next((r for r in rows if r.get("Customer") == customer), None) + + +@router.post("/admin/statements/preview") +def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): + """Render ONE customer's statement exactly as the send path would.""" + cs = _cs() + body = body or {} + rows, _ = _rows() + row = _find(rows, body.get("customer")) + if row is None: + raise err(404, "not_found", "no such customer on the collection list") + t = body.get("templates") or {} + import datetime as dt + month = dt.date.today().strftime("%B %Y") + subject = (t.get("subject") or cs.DEFAULT_SUBJECT) + try: + subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) + except Exception: # noqa: BLE001 + # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they + # can see what they typed rather than getting an opaque error. + # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. + # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a + # TypeError, and every one of them is the same user mistake this branch was written for. + # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the + # one screen somebody opens to avoid mailing 200 people the wrong thing. + pass + return { + "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, + t.get("footer") or cs.DEFAULT_FOOTER), + "to": row.get("Email") or "", + "subject": subject, + } + + +@router.post("/admin/statements/send") +def send(body: dict = Body(default=None), session: Session = Depends(_gate)): + """Queue statements. Returns per-customer outcomes — NEVER a bare count. + + `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the + data layer refuses an address outside the allow-list), which is why the route does not check it. + """ + cs = _cs() + body = body or {} + names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] + if not names: + raise err(400, "bad_request", "name at least one customer") + # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A + # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent + # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) + # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On + # the one route in this product that is allowed to write to Odoo, the difference between a + # rehearsal and mailing a live customer was one empty string. + # + # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` + # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature + # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key + # being present, or `test: true`, is that statement, and an empty address beside it is the + # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` + # asserted with no key at all. + # + # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. + # `automationApi.testSendStatement` refuses an empty address before the request is built, and + # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client + # wall is bypassable by construction (anything can POST); this one is not. + declared_test = ("overrideTo" in body) or bool(body.get("test")) + override = str(body.get("overrideTo") or "").strip() + if declared_test and not override: + raise err(400, "no_override_address", + "a test send needs the address to send the test to. Without one this would " + "mail the customer's own address, which is the opposite of a test") + override = override or None + if override and len(names) != 1: + raise err(400, "bad_request", "a test send takes exactly one customer") + t = body.get("templates") or {} + rows, _ = _rows() + + sent, failed, skipped = [], [], [] + for name in names: + row = _find(rows, name) + if row is None: + failed.append({"customer": name, "error": "not on the current collection list"}) + continue + if not override and not row.get("Email"): + # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures + # keeps "we could not" distinct from "there was nowhere to send". + skipped.append({"customer": name, "reason": "no email address on the customer record"}) + continue + try: + mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT, + t.get("intro") or cs.DEFAULT_INTRO, + t.get("footer") or cs.DEFAULT_FOOTER, override_to=override) + sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid}) + except Exception as e: + # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal + # and an Odoo error are both "this one did not go", each with its own honest message. + failed.append({"customer": name, "error": str(e)[:200]}) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. +# +# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch +# and parks it on the automation definition, and it imports no mail path at all. THIS is the only +# door that releases one, and it is deliberately built out of the parts already here: +# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three +# endpoints use. Not a copy of the checks: the same object. +# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal +# happens in the DATA LAYER, where no route, payload or UI can reach around it. +# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. +# +# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked +# `html` is what a person APPROVED and is what the review screen shows; but a balance can move +# between the parking and the click, and mailing a figure we know to be stale is worse than mailing +# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the +# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely +# (they paid) is reported as skipped rather than invoiced. + +@router.get("/admin/statements/agent/{auto_id}") +def agent_batch(auto_id: str, session: Session = Depends(_gate)): + """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + if not isinstance(pending, dict): + return {"pending": None} + # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is + # megabytes, and the review screen needs the count and the names to decide. `preview` already + # renders ONE on demand. + return {"pending": { + "ts": pending.get("ts"), "count": int(pending.get("count") or 0), + "notes": list(pending.get("notes") or []), + "items": [{k: v for k, v in it.items() if k != "html"} + for it in (pending.get("items") or [])], + "safeMode": bool(_cs().SAFE_MODE)}} + + +@router.post("/admin/statements/agent/{auto_id}/send") +def agent_send(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_gate)): + """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. + + ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send + that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) + arriving on the one route in this system that talks to a mail server. + """ + cs = _cs() + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] + if not items: + raise err(404, "no_batch", "there is nothing parked for this agent to send") + # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it + # may only NARROW the parked batch. A name that was never parked cannot be introduced by the + # payload, or the review stops being what authorised the send. + want = {str(n) for n in (body or {}).get("customers") or []} + if want: + items = [it for it in items if str(it.get("customer")) in want] + if not items: + raise err(400, "bad_request", "none of those customers are in the parked batch") + rows, _ = _rows() + sent, failed, skipped = [], [], [] + for it in items: + name = str(it.get("customer") or "") + row = _find(rows, name) + if row is None: + skipped.append({"customer": name, + "reason": "no longer on the collection list, so nothing is owed"}) + continue + try: + # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE + # `queue_statement`, so the guardrail cannot be argued with from here. + mid = cs.queue_statement(cs.Odoo(), row, + str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, + cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) + sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) + except Exception as e: # noqa: BLE001 + failed.append({"customer": name, "error": str(e)[:200]}) + # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would + # lose the list of who still needs a statement, and nobody would know to look. + if sent and not failed: + engine.clear_statements(session.runtime, auto_id) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} diff --git a/api/routes_tables.py b/api/routes_tables.py index 4a00314fc5471a1b910c22b10e3fdb10d209d663..706e2d8f2e73f2d96a1b4470a03ae7c0ee39e89f 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -1,430 +1,430 @@ -"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). - -The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served -through the SAME grid machinery every other topic rides — `table_store` for the per-user -workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every -durable write except rows. Rows are the one genuinely new channel: the events seam has no row -event types (the user_tables docstring's `row_add` gate was described, never built), so row -add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + -`user_tables.may_open` — a connector-backed table can never accept an invented row. - -TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a -Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 -covers them for free. - -VISIBILITY is `user_tables.may_open` — creator, admin, or a `core.shares` grant, fail-closed, -applied in `_defn_or_refuse` before any payload is built. - -⭐⭐ WAVE 36 (W36-T21 / OWNER RULING R6) — AND IT IS NO LONGER THE WHOLE WALL, WHICH IS THE POINT -OF THE TICKET. That paragraph used to end *"the per-table wall is the whole wall"*, and it was -true: a `ut_*` database was a BINARY door, so an admin could hand somebody all 31,418 rows of -`ut_odoo_invoices` or none of them, while `customer_data` had per-user row filters and hidden -fields. `perms.py`'s own docstring booked the fix and warned what half a fix looks like — *"a -stored `ut_*` wall would be INERT: the editor would say DENY, the table routes would keep serving, -and nothing anywhere would say so."* - -So THREE things are now true of every row this file serves, and each has one place: - * `perm_scope.may_read` — IF: admin, then an explicit stored `access: false`, then `may_open` - UNCHANGED. Composed, never merged. - * `scoped_pool`/`scoped_pids` — WHICH ROWS: the permanent filter, applied BEFORE `pids` is - taken, exactly where `routes_customers.grid_assembly` applies it. - * `ut_assembly` — WHICH FIELDS: the transitive hidden closure, applied AFTER - `workspace_wire`, on BOTH wires (the field list and the row payload). - -⛔ AND A DOOR THAT CANNOT APPLY ANY OF IT REFUSES RATHER THAN SERVING THE LOT — `_defn_or_refuse`'s -`scope_applied` flag, which defaults to fail-closed precisely because the one caller outside this -file cannot pass it. -""" -import threading -import json -import time - +"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). + +The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served +through the SAME grid machinery every other topic rides — `table_store` for the per-user +workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every +durable write except rows. Rows are the one genuinely new channel: the events seam has no row +event types (the user_tables docstring's `row_add` gate was described, never built), so row +add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + +`user_tables.may_open` — a connector-backed table can never accept an invented row. + +TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a +Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 +covers them for free. + +VISIBILITY is `user_tables.may_open` — creator, admin, or a `core.shares` grant, fail-closed, +applied in `_defn_or_refuse` before any payload is built. + +⭐⭐ WAVE 36 (W36-T21 / OWNER RULING R6) — AND IT IS NO LONGER THE WHOLE WALL, WHICH IS THE POINT +OF THE TICKET. That paragraph used to end *"the per-table wall is the whole wall"*, and it was +true: a `ut_*` database was a BINARY door, so an admin could hand somebody all 31,418 rows of +`ut_odoo_invoices` or none of them, while `customer_data` had per-user row filters and hidden +fields. `perms.py`'s own docstring booked the fix and warned what half a fix looks like — *"a +stored `ut_*` wall would be INERT: the editor would say DENY, the table routes would keep serving, +and nothing anywhere would say so."* + +So THREE things are now true of every row this file serves, and each has one place: + * `perm_scope.may_read` — IF: admin, then an explicit stored `access: false`, then `may_open` + UNCHANGED. Composed, never merged. + * `scoped_pool`/`scoped_pids` — WHICH ROWS: the permanent filter, applied BEFORE `pids` is + taken, exactly where `routes_customers.grid_assembly` applies it. + * `ut_assembly` — WHICH FIELDS: the transitive hidden closure, applied AFTER + `workspace_wire`, on BOTH wires (the field list and the row payload). + +⛔ AND A DOOR THAT CANNOT APPLY ANY OF IT REFUSES RATHER THAN SERVING THE LOT — `_defn_or_refuse`'s +`scope_applied` flag, which defaults to fail-closed precisely because the one caller outside this +file cannot pass it. +""" +import threading +import json +import time + from fastapi import Body, Depends, Query -from fastapi import APIRouter - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _ops(session, table_key, st=None): - """This database's per-user workspace store, bound to the tenant. - - ⭐⭐ W36-T24 / D-214 — `st` LETS ONE ASSEMBLY LEND ITS OWN SNAPSHOT, AND THE DOCUMENT IT SAVES - IS `object_shares`, NOT THE WORKSPACE. MEASURED with a call-counting probe over `ut_assembly` - (D-214's own exit condition): one assembly took **5 whole-document reads for an admin and 6 - for a shared, row-scoped user** — `object_shares` TWICE (three times for a non-creator), - `user_tables` once, and `_table_workspace` twice. - - The `object_shares` repeats come from `grid_events._granted_views` and `_granted_folders`, - which ask `shares.shared_with` for the `view` and the `folder` kind separately, each reaching - the store through `tops.st` — this handle. `user_tables.lend` already memoises exactly those - two buckets for one pass (`_LENDABLE`), so handing the ops object a lend collapses them - without touching `core/shares.py`'s semantics or `grid_events`, which is in no wave-36 fence. - - ⛔ WHY IT IS SAFE ON A PATH THAT ALSO WRITES. `_Lent` serves ONLY `user_tables` and - `object_shares`; `_table_workspace` is not lendable, so every workspace read and write - passes straight through to the runtime, and `__getattr__` forwards `update` regardless. The - assembly performs no `object_shares` write, and `patch_row`'s post-write read-back reads - `session.runtime` directly rather than this handle — contract C5's rule, unbroken. - """ - import core.table_store as table_store - return table_store.make(f"{table_key}_table_workspace", - st=st if st is not None else session.runtime) - - -#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF -#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. -_REL_LOCK = threading.Lock() -_REL_DIRTY = {} -_REL_RUNNING = {} - - -def _refresh_relations(session): - """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. - - ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's - item 2 was "adding a new record visually takes too long, I need to be able to spam it", and - this function was the largest single cost inside `POST /tables/{key}/rows`: - `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can - decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something - does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the - default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit - before the 201 came back. - - ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty - whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam - the item asks us to support is exactly the load that would melt it. So this COALESCES: at - most one pass runs, and at most one is queued behind it. - - ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in - flight must still get a pass afterwards, because the in-flight one snapshotted before that - write existed. Without the flag the LAST add in a burst is precisely the one whose rollups - never update — the failure nobody would notice until a total was quietly wrong. - - Eventual consistency is the accepted trade and was already the documented posture: the tick - repairs materialised cells regardless, and answering 503 here would invite the browser to - repeat a mutation that already succeeded. - """ - tenant = str(getattr(session, "tenant", "") or "") - rt = session.runtime - with _REL_LOCK: - _REL_DIRTY[tenant] = True - if _REL_RUNNING.get(tenant): - return # a worker is live; it will see the flag and loop - _REL_RUNNING[tenant] = True - - def _worker(): - import automation_engine as engine - try: - while True: - with _REL_LOCK: - if not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - return - _REL_DIRTY.pop(tenant, None) - try: - engine.refresh_relations(rt, log=lambda *_args: None) - except Exception as exc: # noqa: BLE001 - # The source write already landed and the response is already sent. Log and - # let the tick repair it; never retry in a tight loop. - print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") - finally: - # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked - # RUNNING would never refresh again for the life of the process. - with _REL_LOCK: - if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - - threading.Thread(target=_worker, daemon=True, - name=f"rel-refresh:{tenant or 'default'}").start() - - -def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=False): - """The per-table wall: 404 for a key that does not exist, 403 for one this session may not - open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but - not yours' is exactly what may_open is for. - - ⭐⭐ W36-T21 / R6 / CONTRACT C1 — `scope_applied` IS THE HALF THAT MAKES THE WALL NON-INERT, - AND IT DEFAULTS TO FAIL-CLOSED ON PURPOSE. - - `perms.py`'s own docstring described exactly the defect this parameter prevents: *"a stored - `ut_*` wall would be INERT — the editor would say DENY, the table routes would keep serving, - and nothing anywhere would say so."* So a door that will apply C1's row/field wall to what it - is about to serve says so HERE, in writing, and a door that will not is REFUSED for any - principal carrying a wall on this database (`perm_scope.wall_declared`). - - ⛔ THE DEFAULT IS `False` BECAUSE THE ONE CALLER OUTSIDE THIS FILE CANNOT PASS IT. - `routes_odoo_tables`' windowed rows route calls this guard and then builds its own SQL — it - has no way to apply a row filter or a hidden-field closure, and it is in no wave-36 fence. An - opt-OUT default would have left that door silently serving a walled account the whole table, - which is the INERT wall by another route. Opting IN means a door added tomorrow is refused - until somebody has thought about it ([[default-must-pass-its-own-guard]]). - - ⚠ AND IT COSTS NOTHING FOR EVERYBODY WITH NO WALL — which today is every account in every - tenant, because no `ut_*` wall has ever been storable. `wall_declared` is False for an admin - and False for a record with no entry, so this branch cannot fire until an administrator - deliberately stores one. - - ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the - nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked - about the same table — it is simply not re-reading a 28.5 MB document to ask it. - - ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL - SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open - it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE - DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and - `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the - CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated - with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). - - ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, - `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass - it: a projected snapshot must never reach a post-write read-back (contract C5), and the - pre-write lend note below is the same argument one layer down. - - ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION - DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" - proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate - asserts the fast path was TAKEN. - """ - ut = _ut() - # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. - # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* - # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was - # the route count, not the caller count) and cost TWO whole-document deep copies at every one - # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions - # about ONE table. Lending here fixes every caller at once, including the eight routes the - # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · - # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). - # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The - # guard runs before any mutation and returns only the DEFINITION, so the snapshot never - # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes - # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which - # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it - # replaced. The remaining in-route reads are deliberately untouched and booked instead. - if st is None: - st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) - defn = ut.get(table_key, st=st) - if not defn: - raise err(404, "unknown_table", "that database does not exist") - # ⭐⭐ W36-T21 — ONE evaluator for the IF question, and it CALLS `may_open` rather than - # replacing it. `perm_scope.may_read` is admin -> an explicit stored `access: false` -> - # `user_tables.may_open`, unmodified. Two questions stay two questions: `may_open` still - # decides IF the database is visible, C1 decides WHICH rows and fields. - import core.perm_scope as perm_scope - if not perm_scope.may_read(session.user, table_key, st=st): - raise err(403, "forbidden", "that database belongs to another user") - if not scope_applied and perm_scope.wall_declared(session.user, table_key): - # R6's second sentence: a limit that cannot be met is a SENTENCE naming the cause and a - # fix, never a short answer — and here the short answer would be the WHOLE database. - raise err(409, "scope_not_applied", - "an administrator has restricted which rows and columns of this database you " - "may see, and this view cannot apply that restriction. Open the database from " - "the navigation, where the restriction is applied, or ask an administrator to " - "remove it") - return defn - - -def _records_or_refuse(session, table_key, st=None): - """The human record-write wall for a database the automation engine owns.""" - # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — - # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. - st = st if st is not None else _ut().lend(session.runtime) - # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a - # SCOPED pid set of its own: `patch_row` refuses a pid outside `ut_assembly`'s `pids`, and - # `delete_row` asks the same question just below. Refusing here instead would make a walled - # database READ-ONLY rather than row-scoped, which is a different product. - defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) - if not _ut().records_mutable(table_key, st=st): - raise err(403, "records_read_only", - "records in this automation-owned database are read-only. Add Instagram " - "handles in a Profile database and let enrichment populate this database") - return defn - - -#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT -#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. -#: -#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity -#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never -#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the -#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, -#: and the wall then hid the survivor from the person who just found it. They searched, they -#: paid, and the screen said nothing arrived. -#: -#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named -#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores -#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the -#: write target is whatever database the user points the Create-record action at. So the wall -#: governed a table nobody writes, and fixing EITHER half alone would have armed the other -#: ([[defects-that-mask-each-other]]). -#: -#: The register offered two exits and R4 already implied this one. Restoring per-user visibility -#: instead would have required R4's merge to stop crossing users — a bigger change, against the -#: ruling, to bring back a wall that never governed anything real. -#: -#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every -#: automation-written table has one (a scraped row says `automation`), so inference would hide -#: every scraped row from every non-admin — the same disappearance defect, one table wide instead -#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. - - -def _too_big(): - """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" - import routes_odoo_tables - return routes_odoo_tables.TooBigToMaterialise - - -def _read_through_rows(table_key, field_keys, rt=None): - """The mirror's rows for one read-through grid, projected to this table's declared columns. - - ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, - `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; - the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this - process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, - which is exactly the shape that satisfies the first wall while failing the second. - `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every - tenant, so the two walls are not substitutes for each other. - - ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which - owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror - through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's - by construction rather than by a second query that agrees today. A pid-only `SELECT` would be - cheaper and would also be a SECOND statement of what a row of this table is - ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, - never in how they ask. - - Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides - whether that is a 409 or an unresolved pid scope. - """ - import routes_odoo_tables - - rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} - for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] - rows_src.sort(key=lambda r: r["pid"]) - return rows_src - - -#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. -#: -#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= -#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the -#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. -#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a -#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). -#: -#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is -#: "membership is unresolved on this grid", which is a different claim and has to be made out -#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are -#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` -#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. -_PID_SCOPE_LIMIT = { - "subject": "pids", "effect": "unresolved", - "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " - "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " - "every total is a SQL count over the whole table", -} - - -def scoped_pool(session: Session, table_key: str, st=None): - """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. - - ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as - `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere - else, so the permission question is answered by the same code against the same document. - ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then - reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). - - `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a - caller that only needs "which rows of this database may this session touch" — record comments - — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second - idea of what a user table's pool is. - - ⭐⭐ W36-T21 / R6 — THE ROW WALL RUNS HERE NOW, ON EVERY DATABASE, and the position is the - whole of it: BEFORE `pids` is taken. `routes_customers.grid_assembly` says the same thing in - the same words for `customer_data` — everything downstream is bounded by that frozenset - (`allowed_pids` for the workspace, cohort membership, every write door's pid check), so - scoping here means a row this account may not see never enters ANY of them, rather than being - filtered out of one payload and surviving in another. - - ⚠ THE FIELD HALF IS NOT HERE, and that is parity rather than an omission: the hidden closure - must cover the user's own `custom_`/`measure_` columns, which do not exist until - `workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does. - """ - defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - field_keys = {f["key"] for f in fields_base} - # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. - # - # This is the one function that turns "what is stored" into "what this session may see", which - # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events - # route, the comments wall and the assembly all reach rows through it, so they all convert - # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve - # an EMPTY GRID — correct-looking, wrong, and silent. - # ⚠ The wall above has already run. This adds no scope of its own and takes none away. - import core.perm_scope as perm_scope - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) - except _too_big() as e: - # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. - # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not - # this): a caller that asked for every row of a 963,783-row grid cannot be served a - # short one. `scoped_pids` below takes the same refusal and answers a different - # question with it, because an envelope needs no row. - raise err(409, "window_required", str(e)) - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - # WAVE 27 / D-72: no per-row OWNER filter — the row's creator is not a permission. What - # runs below is a different thing entirely: the permanent filter an ADMIN declared for - # this account (W36-T21 / R6), the same one `grid_assembly` has applied to `customer_data` - # since wave 15. A row this session can reach is a row the tenant owns AND the wall admits. - r = {k: v for k, v in (row or {}).items() if k in field_keys} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - # ⭐⭐ W36-T21 — `permits()`, not `matches()`: an unanswerable permanent filter DENIES rather - # than being ignored. Evaluated against the DECLARED contract (`fields_base`), which is what - # `routes_admin._clean_perms` validates a stored filter against, so the two cannot disagree - # about what a column is. - rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - - -@router.patch("/tables/{table_key}/shared/{pid}") -def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has - been waiting for since it shipped (W29-T62, wave 30 T28). - - ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not - "sharing would be nice": a user-created column and its values live PER USER, so a shared view - filtering on one names a column other accounts do not have — and an unknown column is an - INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy - list would silently show the whole catalogue to everyone but its author. A column whose value - is the same for every reader is the precondition for editing it at all. - - ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no - reader and no writer by design; "may this session open this surface" is answered HERE, where - the session is. Do not push the question down there. - ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. - That is the point, and it is why this door declares the column too — a value with no - definition is a cell nobody can find. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("field") or "").strip() +from fastapi import APIRouter + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _ops(session, table_key, st=None): + """This database's per-user workspace store, bound to the tenant. + + ⭐⭐ W36-T24 / D-214 — `st` LETS ONE ASSEMBLY LEND ITS OWN SNAPSHOT, AND THE DOCUMENT IT SAVES + IS `object_shares`, NOT THE WORKSPACE. MEASURED with a call-counting probe over `ut_assembly` + (D-214's own exit condition): one assembly took **5 whole-document reads for an admin and 6 + for a shared, row-scoped user** — `object_shares` TWICE (three times for a non-creator), + `user_tables` once, and `_table_workspace` twice. + + The `object_shares` repeats come from `grid_events._granted_views` and `_granted_folders`, + which ask `shares.shared_with` for the `view` and the `folder` kind separately, each reaching + the store through `tops.st` — this handle. `user_tables.lend` already memoises exactly those + two buckets for one pass (`_LENDABLE`), so handing the ops object a lend collapses them + without touching `core/shares.py`'s semantics or `grid_events`, which is in no wave-36 fence. + + ⛔ WHY IT IS SAFE ON A PATH THAT ALSO WRITES. `_Lent` serves ONLY `user_tables` and + `object_shares`; `_table_workspace` is not lendable, so every workspace read and write + passes straight through to the runtime, and `__getattr__` forwards `update` regardless. The + assembly performs no `object_shares` write, and `patch_row`'s post-write read-back reads + `session.runtime` directly rather than this handle — contract C5's rule, unbroken. + """ + import core.table_store as table_store + return table_store.make(f"{table_key}_table_workspace", + st=st if st is not None else session.runtime) + + +#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF +#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. +_REL_LOCK = threading.Lock() +_REL_DIRTY = {} +_REL_RUNNING = {} + + +def _refresh_relations(session): + """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. + + ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's + item 2 was "adding a new record visually takes too long, I need to be able to spam it", and + this function was the largest single cost inside `POST /tables/{key}/rows`: + `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can + decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something + does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the + default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit + before the 201 came back. + + ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty + whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam + the item asks us to support is exactly the load that would melt it. So this COALESCES: at + most one pass runs, and at most one is queued behind it. + + ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in + flight must still get a pass afterwards, because the in-flight one snapshotted before that + write existed. Without the flag the LAST add in a burst is precisely the one whose rollups + never update — the failure nobody would notice until a total was quietly wrong. + + Eventual consistency is the accepted trade and was already the documented posture: the tick + repairs materialised cells regardless, and answering 503 here would invite the browser to + repeat a mutation that already succeeded. + """ + tenant = str(getattr(session, "tenant", "") or "") + rt = session.runtime + with _REL_LOCK: + _REL_DIRTY[tenant] = True + if _REL_RUNNING.get(tenant): + return # a worker is live; it will see the flag and loop + _REL_RUNNING[tenant] = True + + def _worker(): + import automation_engine as engine + try: + while True: + with _REL_LOCK: + if not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + return + _REL_DIRTY.pop(tenant, None) + try: + engine.refresh_relations(rt, log=lambda *_args: None) + except Exception as exc: # noqa: BLE001 + # The source write already landed and the response is already sent. Log and + # let the tick repair it; never retry in a tight loop. + print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") + finally: + # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked + # RUNNING would never refresh again for the life of the process. + with _REL_LOCK: + if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + + threading.Thread(target=_worker, daemon=True, + name=f"rel-refresh:{tenant or 'default'}").start() + + +def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=False): + """The per-table wall: 404 for a key that does not exist, 403 for one this session may not + open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but + not yours' is exactly what may_open is for. + + ⭐⭐ W36-T21 / R6 / CONTRACT C1 — `scope_applied` IS THE HALF THAT MAKES THE WALL NON-INERT, + AND IT DEFAULTS TO FAIL-CLOSED ON PURPOSE. + + `perms.py`'s own docstring described exactly the defect this parameter prevents: *"a stored + `ut_*` wall would be INERT — the editor would say DENY, the table routes would keep serving, + and nothing anywhere would say so."* So a door that will apply C1's row/field wall to what it + is about to serve says so HERE, in writing, and a door that will not is REFUSED for any + principal carrying a wall on this database (`perm_scope.wall_declared`). + + ⛔ THE DEFAULT IS `False` BECAUSE THE ONE CALLER OUTSIDE THIS FILE CANNOT PASS IT. + `routes_odoo_tables`' windowed rows route calls this guard and then builds its own SQL — it + has no way to apply a row filter or a hidden-field closure, and it is in no wave-36 fence. An + opt-OUT default would have left that door silently serving a walled account the whole table, + which is the INERT wall by another route. Opting IN means a door added tomorrow is refused + until somebody has thought about it ([[default-must-pass-its-own-guard]]). + + ⚠ AND IT COSTS NOTHING FOR EVERYBODY WITH NO WALL — which today is every account in every + tenant, because no `ut_*` wall has ever been storable. `wall_declared` is False for an admin + and False for a record with no entry, so this branch cannot fire until an administrator + deliberately stores one. + + ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the + nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked + about the same table — it is simply not re-reading a 28.5 MB document to ask it. + + ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL + SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open + it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE + DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and + `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the + CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated + with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). + + ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, + `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass + it: a projected snapshot must never reach a post-write read-back (contract C5), and the + pre-write lend note below is the same argument one layer down. + + ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION + DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" + proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate + asserts the fast path was TAKEN. + """ + ut = _ut() + # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. + # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* + # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was + # the route count, not the caller count) and cost TWO whole-document deep copies at every one + # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions + # about ONE table. Lending here fixes every caller at once, including the eight routes the + # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · + # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). + # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The + # guard runs before any mutation and returns only the DEFINITION, so the snapshot never + # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes + # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which + # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it + # replaced. The remaining in-route reads are deliberately untouched and booked instead. + if st is None: + st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) + defn = ut.get(table_key, st=st) + if not defn: + raise err(404, "unknown_table", "that database does not exist") + # ⭐⭐ W36-T21 — ONE evaluator for the IF question, and it CALLS `may_open` rather than + # replacing it. `perm_scope.may_read` is admin -> an explicit stored `access: false` -> + # `user_tables.may_open`, unmodified. Two questions stay two questions: `may_open` still + # decides IF the database is visible, C1 decides WHICH rows and fields. + import core.perm_scope as perm_scope + if not perm_scope.may_read(session.user, table_key, st=st): + raise err(403, "forbidden", "that database belongs to another user") + if not scope_applied and perm_scope.wall_declared(session.user, table_key): + # R6's second sentence: a limit that cannot be met is a SENTENCE naming the cause and a + # fix, never a short answer — and here the short answer would be the WHOLE database. + raise err(409, "scope_not_applied", + "an administrator has restricted which rows and columns of this database you " + "may see, and this view cannot apply that restriction. Open the database from " + "the navigation, where the restriction is applied, or ask an administrator to " + "remove it") + return defn + + +def _records_or_refuse(session, table_key, st=None): + """The human record-write wall for a database the automation engine owns.""" + # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — + # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. + st = st if st is not None else _ut().lend(session.runtime) + # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a + # SCOPED pid set of its own: `patch_row` refuses a pid outside `ut_assembly`'s `pids`, and + # `delete_row` asks the same question just below. Refusing here instead would make a walled + # database READ-ONLY rather than row-scoped, which is a different product. + defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) + if not _ut().records_mutable(table_key, st=st): + raise err(403, "records_read_only", + "records in this automation-owned database are read-only. Add Instagram " + "handles in a Profile database and let enrichment populate this database") + return defn + + +#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT +#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. +#: +#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity +#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never +#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the +#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, +#: and the wall then hid the survivor from the person who just found it. They searched, they +#: paid, and the screen said nothing arrived. +#: +#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named +#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores +#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the +#: write target is whatever database the user points the Create-record action at. So the wall +#: governed a table nobody writes, and fixing EITHER half alone would have armed the other +#: ([[defects-that-mask-each-other]]). +#: +#: The register offered two exits and R4 already implied this one. Restoring per-user visibility +#: instead would have required R4's merge to stop crossing users — a bigger change, against the +#: ruling, to bring back a wall that never governed anything real. +#: +#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every +#: automation-written table has one (a scraped row says `automation`), so inference would hide +#: every scraped row from every non-admin — the same disappearance defect, one table wide instead +#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. + + +def _too_big(): + """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" + import routes_odoo_tables + return routes_odoo_tables.TooBigToMaterialise + + +def _read_through_rows(table_key, field_keys, rt=None): + """The mirror's rows for one read-through grid, projected to this table's declared columns. + + ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, + `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; + the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this + process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, + which is exactly the shape that satisfies the first wall while failing the second. + `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every + tenant, so the two walls are not substitutes for each other. + + ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which + owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror + through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's + by construction rather than by a second query that agrees today. A pid-only `SELECT` would be + cheaper and would also be a SECOND statement of what a row of this table is + ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, + never in how they ask. + + Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides + whether that is a 409 or an unresolved pid scope. + """ + import routes_odoo_tables + + rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} + for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] + rows_src.sort(key=lambda r: r["pid"]) + return rows_src + + +#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. +#: +#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= +#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the +#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. +#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a +#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). +#: +#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is +#: "membership is unresolved on this grid", which is a different claim and has to be made out +#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are +#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` +#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. +_PID_SCOPE_LIMIT = { + "subject": "pids", "effect": "unresolved", + "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " + "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " + "every total is a SQL count over the whole table", +} + + +def scoped_pool(session: Session, table_key: str, st=None): + """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. + + ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as + `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere + else, so the permission question is answered by the same code against the same document. + ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then + reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). + + `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a + caller that only needs "which rows of this database may this session touch" — record comments + — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second + idea of what a user table's pool is. + + ⭐⭐ W36-T21 / R6 — THE ROW WALL RUNS HERE NOW, ON EVERY DATABASE, and the position is the + whole of it: BEFORE `pids` is taken. `routes_customers.grid_assembly` says the same thing in + the same words for `customer_data` — everything downstream is bounded by that frozenset + (`allowed_pids` for the workspace, cohort membership, every write door's pid check), so + scoping here means a row this account may not see never enters ANY of them, rather than being + filtered out of one payload and surviving in another. + + ⚠ THE FIELD HALF IS NOT HERE, and that is parity rather than an omission: the hidden closure + must cover the user's own `custom_`/`measure_` columns, which do not exist until + `workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does. + """ + defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + field_keys = {f["key"] for f in fields_base} + # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. + # + # This is the one function that turns "what is stored" into "what this session may see", which + # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events + # route, the comments wall and the assembly all reach rows through it, so they all convert + # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve + # an EMPTY GRID — correct-looking, wrong, and silent. + # ⚠ The wall above has already run. This adds no scope of its own and takes none away. + import core.perm_scope as perm_scope + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) + except _too_big() as e: + # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. + # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not + # this): a caller that asked for every row of a 963,783-row grid cannot be served a + # short one. `scoped_pids` below takes the same refusal and answers a different + # question with it, because an envelope needs no row. + raise err(409, "window_required", str(e)) + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + # WAVE 27 / D-72: no per-row OWNER filter — the row's creator is not a permission. What + # runs below is a different thing entirely: the permanent filter an ADMIN declared for + # this account (W36-T21 / R6), the same one `grid_assembly` has applied to `customer_data` + # since wave 15. A row this session can reach is a row the tenant owns AND the wall admits. + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + # ⭐⭐ W36-T21 — `permits()`, not `matches()`: an unanswerable permanent filter DENIES rather + # than being ignored. Evaluated against the DECLARED contract (`fields_base`), which is what + # `routes_admin._clean_perms` validates a stored filter against, so the two cannot disagree + # about what a column is. + rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + + +@router.patch("/tables/{table_key}/shared/{pid}") +def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has + been waiting for since it shipped (W29-T62, wave 30 T28). + + ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not + "sharing would be nice": a user-created column and its values live PER USER, so a shared view + filtering on one names a column other accounts do not have — and an unknown column is an + INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy + list would silently show the whole catalogue to everyone but its author. A column whose value + is the same for every reader is the precondition for editing it at all. + + ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no + reader and no writer by design; "may this session open this surface" is answered HERE, where + the session is. Do not push the question down there. + ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. + That is the point, and it is why this door declares the column too — a value with no + definition is a cell nobody can find. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("field") or "").strip() if not key: raise err(400, "bad_request", "a field key is required") _defn_or_refuse(session, table_key) @@ -434,316 +434,396 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), raise err(403, "forbidden", "an edit share on this database is required to write a tenant-wide cell") from core import shared_overlay - import core.perm_scope as perm_scope - if not shared_overlay.is_shared(table_key, key, st=session.runtime): - # ⭐⭐ W38-T16 / R7 — A COLUMN IS BORN OWNED AND PRIVATE, AND THIS IS THE PRODUCT CHANGE. - # Until now this door minted a column every account in the tenant could read, with no way - # to say who. `granted` marks it as governed by `core.shares` (see - # `perm_scope.FIELD_GRANT_MARK` for why an explicit marker and not the absence of a grant - # record), and the grant record below claims it for the creator. - # - # ⚠ THE EMPTY ENTRY LIST IS THE POINT, NOT A PLACEHOLDER. `set_grants` keeps a record that - # has an owner and no entries, so "shared with nobody" is STORED and is a different fact - # from "never shared" — the same explicit-resolution shape `perms_v` uses. Without the - # owner the column would be unmanageable: `may_administer` fails closed on an ownerless - # record, so nobody could ever share or re-share it. - # ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading - # their absence as "granted to nobody" would blank every existing shared column at once. - shared_overlay.put_field(table_key, key, { - "key": key, "label": str(body.get("label") or key), "source": "overlay", - "type": str(body.get("type") or "text"), "shared": True, - perm_scope.FIELD_GRANT_MARK: True, - "createdBy": session.uname}, st=session.runtime) - try: - import core.shares as shares - shares.set_grants("field", shares.field_oid(table_key, key), [], - owner=session.uname, st=session.runtime) - except Exception: # noqa: BLE001 - # ⛔ THE MARK IS ALREADY WRITTEN, SO A FAILED CLAIM FAILS **CLOSED**: the column is - # governed and nobody holds a grant, i.e. only an admin sees it. That is recoverable - # (an admin can share it) and the other order is not — a marked column with a claim - # that landed first and a definition that did not would be a grant on nothing. - pass - try: - # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is - # the API that says so. It delegates to the plural, so both stay reachable through the one - # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 - # named it (the same lens that found `drop_field` had no door either). - stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), - st=session.runtime)} - except ValueError as e: - # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. - raise err(400, "bad_value", str(e)) - return {"ok": True, "pid": pid, "cells": stored, - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -@router.delete("/tables/{table_key}/shared/fields/{field_key}") -def delete_shared_field(table_key: str, field_key: str, - session: Session = Depends(require_session)): - """Remove a TENANT-WIDE column and every value in it. - - ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column - and none that removes one, so a column anybody added was permanent for the whole tenant. The - reachability gate found it from the other end — `shared_overlay.drop_field` was complete, - correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). - - ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a - value; dropping the column deletes that value for EVERY account at once, so it is the - destructive-op wall this repo already uses for a database delete — not `editRole`, which - governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). - ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed - is admin-only, which is the safe direction. - """ - _defn_or_refuse(session, table_key) - from core import shared_overlay - defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) - if not defn: - raise err(404, "unknown_field", "that column is not a shared column on this database") - owner = str(defn.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a tenant-wide column can be removed by its creator or an admin. This one " - f"was added by {owner or 'somebody else'}, and dropping it would delete the " - f"value for every account") - dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) - # ⭐⭐ W38-T16 — THE GRANTS DIE WITH THE COLUMN, which is `shares.drop_objects`' whole reason - # for existing (wave 21, C3): a deleted object's grants would otherwise serve a ghost id into - # every receiver's "Shared with me" forever, and the ghost 404s on open. It matters twice as - # much for a field, because `drop_field` scrubs the CELLS so the key can be re-used — and a - # surviving grant record would silently re-arm on the next column that took the name. - # ⚠ R8 / D-172 IS UNCHANGED ABOVE: delete is creator-or-admin. This ticket did not widen it. - try: - import core.shares as shares - shares.drop_objects([("field", shares.field_oid(table_key, str(field_key)))], - st=session.runtime) - except Exception: # noqa: BLE001 - pass - return {"ok": True, "dropped": bool(dropped), - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -def scoped_pids(session: Session, table_key: str, limits=None, st=None): - """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without - building a row. - - ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write - (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate - it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on - every toggle — and the six keys the events route actually reads from that assembly - (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. - `rows_src` was computed and discarded. - - ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: - `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the - row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the - middle. The row WALL is unchanged — a narrower or wider set here would be a permission - change, and this is a performance change. - - ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document - read, which is D-87 and W30-T31. This removes the row pass. - ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then - `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions - have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a - PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. - Booked because a stale performance note is how a wave re-fixes something twice - ([[stale-baseline-unreadable-deltas]]). - - ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a - list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a - read-through grid whose population exceeds one window). The pid set is then EMPTY, and every - consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the - silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a - write is expected to carry the sentence through. Omitting the list means the caller accepts an - unannounced empty scope, which is only ever right for a caller that does not use the pids. - """ - # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. - # - # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` - # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a - # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document - # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` - # off the definition — no row — so the WALL can be answered from the 0.1% projection. - # - # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED - # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. - # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads - # `readThrough` — a definition key, safe under the projection, and already lent the defn so it - # costs no read of its own. - import core.perm_scope as perm_scope - # ⭐ W36-T24 / D-214 — `st` LETS THE ASSEMBLY LEND ITS OWN PASS, exactly as `scoped_pool` has - # since W33-T03. ⛔ It must be a PROJECTED lend (`lend_defs`), not a whole one: the saving - # D-213 bought on the database-switch path is that this wall answers from the 0.1% document, - # and handing it `lend()` would quietly take that back while looking like an optimisation. - defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the - # comprehension below would answer an EMPTY pid set — and the promise this function makes is - # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror - # through the SAME fetch that function uses rather than growing a second idea of the row set; - # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, - rt=session.runtime) - rows = perm_scope.apply_row_scope(rows, session.user, table_key, fields_base) - return frozenset(r["pid"] for r in rows), fields_base, defn - except _too_big() as e: - # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning - # this into a 409 is what made both line grids unopenable: the envelope was refused - # over rows it never renders. The scope is empty and SAID to be empty. - if limits is not None: - limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) - return frozenset(), fields_base, defn - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection - # above cannot serve it and would raise rather than answer `{}` (that is the whole design of - # `_Projected`). The wall has already passed on the projected document, so this re-reads the - # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped - # answer to a question already answered, which is how two ideas of ownership got into this file - # once before (see `may_open`'s own note in `core/user_tables.py`). - # - # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection - # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the - # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten - # read-through databases (every one of them, which is why the switch was slow) skip the big - # read entirely. - whole = _ut().get(table_key, st=session.runtime) - if whole is None: - # Between the wall and here the table was deleted by another request. Same refusal the - # wall gives, rather than an empty pid set nobody can distinguish from an empty table. - raise err(404, "unknown_table", "that database does not exist") - # ⭐⭐ W36-T21 — AND THE ROW WALL, WHICH IS WHY THIS ARM CAN NO LONGER ALWAYS SKIP THE ROWS. - # - # ⛔ THE PROMISE THIS FUNCTION MAKES IS THAT ITS PID SET IS **IDENTICAL** TO `scoped_pool`'s, - # not merely cheaper. `scoped_pool` now narrows its rows by the permanent filter before taking - # pids, so a set built here from the raw row ids would be WIDER — and every consumer of these - # pids (the workspace envelope, cohort membership, `patch_row`'s scope check) would admit rows - # the read door refuses. That is two ideas of one row set, which is the exact defect class - # `may_open`'s own wave-20 note records ([[one-question-two-normalizers]]). - # - # ⭐ AND W30-T30's SAVING SURVIVES FOR EVERYBODY IT WAS FOR. `row_scope_applies` is False for - # an admin and for every record with no declared filter, which is every account in every - # tenant today — those callers take the id comprehension exactly as before and build no row. - # Only a principal an administrator has actually row-scoped pays the pass, and for them the - # alternative is not "cheaper" but "wrong". - rows = whole.get("rows") or {} - if not perm_scope.row_scope_applies(session.user, table_key): - return frozenset(int(rid) for rid in rows if str(rid).isdigit()), fields_base, whole - keys = {f["key"] for f in fields_base if f.get("key")} - scoped = perm_scope.apply_row_scope( - [{**{k: v for k, v in (row or {}).items() if k in keys}, "pid": int(rid)} - for rid, row in rows.items() if str(rid).isdigit()], - session.user, table_key, fields_base) - return frozenset(r["pid"] for r in scoped), fields_base, whole - - -def ut_write_ctx(session: Session, table_key: str): - """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. - - Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this - or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller - that starts needing rows should fail on an empty list it can see, not on a KeyError. - """ - import aios_grid - from core import grid_events - - limits = [] - # ⭐ W36-T24 / D-214 — ONE lend for the whole pass, so the wall and the grant legs stop - # reading `object_shares` once each. Projected, because this ctx reads no row either. - lent = _ut().lend_defs(session.runtime) - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=lent) - # ⭐⭐ W36-T21 — THE FIELD WALL ON THE **WRITE** CTX, and it is not a copy of the read one. - # `grid_events` refuses a hidden key by asking `ctx.hidden_keys` (`grid_events.py:1802` and - # `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully - # writable on the EVENTS transport — a wall on one wire and not the other is the shape - # `strip_row`'s own note warns about, with the sign flipped. - hidden = _ut_hidden(session, table_key, fields_base, st=lent) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, - admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, - scope_key=table_key, table=_ops(session, table_key, st=lent)) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", - fields_base=fields_base) - # ⭐⭐ W38-T16 — THE SHARED COLUMNS JOIN THE **WRITE** CTX'S CONTRACT TOO, and for the reason - # W36-T21 gave one wall over: `grid_events` refuses a key by asking `ctx.hidden_keys`, so a - # column missing from this list is a column the events transport does not know it must - # refuse. The read door narrowing a name the write door has never heard of is the same - # one-wire wall with the sign flipped. - fields = _ut_shared_fields(session, table_key, fields) - fields, _rows, hidden = _ut_field_wall(session, table_key, fields, [], st=lent) - return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, "hidden": hidden, - "derived": aios_grid.cohort_cells(lists), - # ⭐ W37-T12 — the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION - # list for `clean_measure_field`: without it a Metric create on this surface is - # refused `measure_not_offered` even though the read door offers the kind. - # ⛔ THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and - # materialises no rows at all (W30-T30 removed a full table build from the write - # path); computing measure values would put that work straight back. `measure_sets` - # stays empty for the same reason — a write is not a render. - # ⭐ AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN - # ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have - # different jobs: `measure_keys` (from `measures`) is the ADMISSION list - # `clean_filter_tree` uses — non-empty here, so a saved measure CONDITION survives the - # write, which is the failure that comment records for the customer path. Empty - # `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender", - # so the next `/workspace` — which DOES compute the sets, in `ut_assembly` — resolves - # it. Conservative in the right direction: one extra repaint, never a dropped filter. - # ⭐⭐ W38-T19 — THE CAPABILITY IS APPLIED HERE TOO, and this is the site that makes - # the refusal REAL rather than cosmetic: `routes_grid` reads this list as - # `measure_offer`, and `clean_measure_field` fail-closes `measure_not_offered` over - # an empty one. A picker hidden on the read door with this list still full would be - # a control that only stops the honest. - "measures": ut_measures(table_key, session), - "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), - # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather - # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. - "limits": limits, "defn": defn} - - -def _ut_hidden(session, table_key, fields, st=None): - """The hidden-field closure for THIS session on THIS database — C1's field half, once. - - ⚠ Named rather than inlined at its four call sites for the reason `may_open`'s own note gives: - four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE - evaluator; this is just the `ut_*` caller's shorthand for it. - - ⭐ W38-T16 — `st` IS THE LEND THE CALLER ALREADY HOLDS, and passing it is free rather than - merely tidy: the closure's field-grant leg resolves against `object_shares`, which is exactly - the bucket `ut_assembly`'s own lend note says the pass is already serving. - """ - import core.perm_scope as perm_scope - return perm_scope.hidden_keys(session.user, table_key, fields, st=st) - - -def _ut_shared_fields(session, table_key, fields): - """`fields` PLUS this database's TENANT-WIDE columns — the read-side merge, W38-T16. - - ⛔⛔ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into - `__shared` for any `ut_*` database since W30-T28, and only the READ-THROUGH door - (`routes_odoo_tables`) ever merged that stratum back. So a shared column written on a - MATERIALISED user table went into the store and was readable by nobody, through any door — - complete, correct, and invisible ([[reachable-is-not-the-same-as-built]]). This is the other - half of that door. - - ⚠ MERGED BEFORE THE WALL, NEVER AFTER, and `routes_odoo_tables` learned this the same way: - the hidden closure must run on the WHOLE contract, or a formula in a shared column that reads - a hidden one sits outside its reach and carries the hidden value out wearing a second name. - ⚠ A key the definition already declares WINS. A shared column is an addition to a database's - contract, never a redefinition of a column that database already has. - """ + import core.perm_scope as perm_scope + if not shared_overlay.is_shared(table_key, key, st=session.runtime): + # ⭐⭐ W38-T16 / R7 — A COLUMN IS BORN OWNED AND PRIVATE, AND THIS IS THE PRODUCT CHANGE. + # Until now this door minted a column every account in the tenant could read, with no way + # to say who. `granted` marks it as governed by `core.shares` (see + # `perm_scope.FIELD_GRANT_MARK` for why an explicit marker and not the absence of a grant + # record), and the grant record below claims it for the creator. + # + # ⚠ THE EMPTY ENTRY LIST IS THE POINT, NOT A PLACEHOLDER. `set_grants` keeps a record that + # has an owner and no entries, so "shared with nobody" is STORED and is a different fact + # from "never shared" — the same explicit-resolution shape `perms_v` uses. Without the + # owner the column would be unmanageable: `may_administer` fails closed on an ownerless + # record, so nobody could ever share or re-share it. + # ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading + # their absence as "granted to nobody" would blank every existing shared column at once. + shared_overlay.put_field(table_key, key, { + "key": key, "label": str(body.get("label") or key), "source": "overlay", + "type": str(body.get("type") or "text"), "shared": True, + perm_scope.FIELD_GRANT_MARK: True, + "createdBy": session.uname}, st=session.runtime) + try: + import core.shares as shares + shares.set_grants("field", shares.field_oid(table_key, key), [], + owner=session.uname, st=session.runtime) + except Exception: # noqa: BLE001 + # ⛔ THE MARK IS ALREADY WRITTEN, SO A FAILED CLAIM FAILS **CLOSED**: the column is + # governed and nobody holds a grant, i.e. only an admin sees it. That is recoverable + # (an admin can share it) and the other order is not — a marked column with a claim + # that landed first and a definition that did not would be a grant on nothing. + pass + try: + # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is + # the API that says so. It delegates to the plural, so both stay reachable through the one + # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 + # named it (the same lens that found `drop_field` had no door either). + stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), + st=session.runtime)} + except ValueError as e: + # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. + raise err(400, "bad_value", str(e)) + return {"ok": True, "pid": pid, "cells": stored, + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +def _delete_field_sentence(code, owner): + """⭐⭐ W41-T07 / CONTRACT C2 — ONE REFUSAL CODE -> THE SENTENCE **THIS DOOR** SAYS. + + C2's last sentence is *"No surface re-implements either test"*, and this function does not: + the TEST is `user_tables.delete_field_refusal`, which returns a stable identifier + (`preset_not_deletable`, `not_field_owner`, `field_not_found`) and no prose at all. Turning + that identifier into a sentence is a RENDERING decision, and rendering belongs to the surface + — the same split `grid_events.EventResult.refusals` already ships, where `code` is the machine + token beside a separate `message`. Printing the code itself would put an identifier on a + screen, which is what T02's own note says the codes exist to prevent. + + ⛔ THE `not_field_owner` SENTENCE IS BYTE-IDENTICAL TO THE ONE THIS DOOR SAID BEFORE THE + PREDICATE EXISTED, and that is not sentiment. It NAMES the person who owns the column, and + `verify_scopes` asserts exactly that (*"...and the refusal NAMES who owns it, rather than just + saying no"*). A refusal a reader cannot act on is barely better than silence; "ask Karen" is + an action and "forbidden" is not. + + ⛔ CONTRACT C7 — NO EM DASH AND NO EN DASH IN ANY OF THESE STRINGS. They are user-facing copy + the moment they reach an error toast, and this ticket ADDS one of them. Rewritten to read + naturally without the punctuation rather than having it stripped out. + """ + ut = _ut() + if code == ut.PRESET_NOT_DELETABLE: + # R6a: *"A pre-set is never deletable, by anyone, including admins."* The sentence has to + # carry that, or an administrator reads "forbidden" and goes looking for the permission + # they are missing. There is no such permission, so it names the way out instead. + return ("this column belongs to the database itself, so it cannot be deleted by anyone, " + "administrators included. Hide it if you do not want to see it") + if code == ut.NOT_FIELD_OWNER: + return (f"a tenant-wide column can be removed by its creator or an admin. This one " + f"was added by {owner or 'somebody else'}, and dropping it would delete the " + f"value for every account") + return ("that column could not be removed. Reload the database and try again, or ask an " + "administrator to remove it") + + +@router.delete("/tables/{table_key}/shared/fields/{field_key}") +def delete_shared_field(table_key: str, field_key: str, + session: Session = Depends(require_session)): + """Remove a TENANT-WIDE column and every value in it. + + ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column + and none that removes one, so a column anybody added was permanent for the whole tenant. The + reachability gate found it from the other end — `shared_overlay.drop_field` was complete, + correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). + + ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a + value; dropping the column deletes that value for EVERY account at once, so it is the + destructive-op wall this repo already uses for a database delete — not `editRole`, which + governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). + ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed + is admin-only, which is the safe direction. + """ + _defn_or_refuse(session, table_key) + from core import field_permissions, shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) + if not defn: + raise err(404, "unknown_field", "that column is not a shared column on this database") + # ⭐⭐ W41-T07 / CONTRACT C2 — THE WALL IS THE ONE PAIR'S, NOT THIS DOOR'S ANY MORE. + # + # This function used to spell the creator-or-admin test itself (`not session.admin and owner + # != session.uname`). It was RIGHT, and being right in a second place is exactly what C2 + # forbids: `grid_events.field_delete` had no test at all until W41-T02, and the way the two + # doors came apart is that each owned its own idea of the rule. `delete_field_refusal` answers + # both halves in one call — the decision (`is None`) and the reason a person reads. + # + # ⛔ AND IT IS STRICTLY NARROWER THAN THE LINE IT REPLACES, IN ONE DIRECTION WORTH NAMING: R6a + # puts the pre-set refusal ABOVE the admin branch, so an administrator is now refused a + # pre-set delete. `field_origin` reads a column with neither `custom: True` nor `createdBy` as + # `preset`, and a shared column minted before W38-T16 stamped `createdBy` carries neither, so + # such a column becomes undeletable rather than admin-only. That is the fail-closed direction + # and it is C2's rule rather than this door's judgment, but it is a real capability change and + # it is booked as a defect rather than left to be discovered. + # + # ⚠ `field=defn` IS PASSED BECAUSE THE PAIR CANNOT FIND THIS COLUMN ON ITS OWN. Without it + # `_field_badges` falls back to `user_tables.get(table_key)`, which reads the DATABASE's + # contract; a tenant-wide column lives in `__shared`, a different document, so every + # delete would answer `field_not_found` and 404. `grant_topic` is the registry namespace the + # mint door used (`patch_shared_cell` writes `shares.field_oid(table_key, key)`), which for a + # `ut_*` database is the bare table key. + # + # ⛔ AND `field_not_found` IS A REFUSAL HERE, NOT A 404, WHICH LOOKS BACKWARDS AND IS NOT. The + # 404 above has already established that this column IS in the shared bucket, and the + # definition it found is what gets handed to the predicate — so the only way the pair can + # still answer `field_not_found` is `_field_badges` failing to CLASSIFY it (its own note: a + # store blip returns None, "fail-closed in the direction that keeps a column alive"). Turning + # that into a 404 would tell the reader the column does not exist while it sits on their + # screen, and would answer the destructive request with something other than "no". + ut = _ut() + refusal = ut.delete_field_refusal( + table_key, str(field_key), session.uname, session.admin, session.runtime, + field=defn, grant_topic=table_key) + if refusal is not None: + raise err(403, "forbidden", + _delete_field_sentence(refusal, str(defn.get("createdBy") or ""))) + # ⭐⭐ W41-T07 — AND THE DELETE IS NOW ALL FIVE LEGS, THROUGH THE ONE EXECUTOR. + # + # ⛔ WHAT THIS DOOR USED TO DO WAS LEG 2 AND LEG 4 OF FIVE. `shared_overlay.drop_field` plus + # `shares.drop_objects` removes the tenant-wide definition, its cells and its grants, and + # leaves every per-user FORK of the same key sitting in `_table_workspace` — where + # `field_permissions.migrate_legacy_fields`, which runs on EVERY read, promotes one straight + # back into the bucket this door just emptied. Measured in-process: drop, one read, the column + # is back. That is the owner's *"reappeared three times"*, and no amount of care at this layer + # could have fixed it, because the residue is in a document this door never opened. + # + # ⚠ THE GRANT DROP MOVED INTO THE EXECUTOR RATHER THAN BEING DELETED. W38-T16's reason for it + # is unchanged and is worth keeping written down: *"a deleted object's grants would otherwise + # serve a ghost id into every receiver's 'Shared with me' forever"*, and it matters twice over + # for a field because the key can be re-used and a surviving record would re-arm on whatever + # column takes the name next. It is now leg 4 of one operation instead of a second write this + # door remembers to make. + removed = field_permissions.delete_field_everywhere( + f"{table_key}_table_workspace", table_key, table_key, str(field_key), + st=session.runtime) + # ⭐ THE RESULT IS TRUTHY AND IT SAYS WHAT WENT, per this ticket's `done-when`. `dropped` keeps + # its name and its meaning for the gate that reads it; `removed` is the per-leg report, so a + # delete that reached the shared bucket but no fork (or the reverse) is legible in the response + # instead of being a bare `true`. `rerender` is the client's instruction to refetch rather than + # trust its optimistic removal, which is the half of the defect that let a stale column survive + # a whole session. + return {"ok": True, "dropped": bool(removed.get("shared") or removed.get("personal")), + "rerender": True, "removed": removed, + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +def scoped_pids(session: Session, table_key: str, limits=None, st=None): + """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without + building a row. + + ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write + (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate + it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on + every toggle — and the six keys the events route actually reads from that assembly + (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. + `rows_src` was computed and discarded. + + ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: + `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the + row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the + middle. The row WALL is unchanged — a narrower or wider set here would be a permission + change, and this is a performance change. + + ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document + read, which is D-87 and W30-T31. This removes the row pass. + ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then + `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions + have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a + PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. + Booked because a stale performance note is how a wave re-fixes something twice + ([[stale-baseline-unreadable-deltas]]). + + ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a + list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a + read-through grid whose population exceeds one window). The pid set is then EMPTY, and every + consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the + silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a + write is expected to carry the sentence through. Omitting the list means the caller accepts an + unannounced empty scope, which is only ever right for a caller that does not use the pids. + """ + # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. + # + # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` + # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a + # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document + # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` + # off the definition — no row — so the WALL can be answered from the 0.1% projection. + # + # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED + # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. + # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads + # `readThrough` — a definition key, safe under the projection, and already lent the defn so it + # costs no read of its own. + import core.perm_scope as perm_scope + # ⭐ W36-T24 / D-214 — `st` LETS THE ASSEMBLY LEND ITS OWN PASS, exactly as `scoped_pool` has + # since W33-T03. ⛔ It must be a PROJECTED lend (`lend_defs`), not a whole one: the saving + # D-213 bought on the database-switch path is that this wall answers from the 0.1% document, + # and handing it `lend()` would quietly take that back while looking like an optimisation. + defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the + # comprehension below would answer an EMPTY pid set — and the promise this function makes is + # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror + # through the SAME fetch that function uses rather than growing a second idea of the row set; + # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, + rt=session.runtime) + rows = perm_scope.apply_row_scope(rows, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows), fields_base, defn + except _too_big() as e: + # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning + # this into a 409 is what made both line grids unopenable: the envelope was refused + # over rows it never renders. The scope is empty and SAID to be empty. + if limits is not None: + limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) + return frozenset(), fields_base, defn + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection + # above cannot serve it and would raise rather than answer `{}` (that is the whole design of + # `_Projected`). The wall has already passed on the projected document, so this re-reads the + # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped + # answer to a question already answered, which is how two ideas of ownership got into this file + # once before (see `may_open`'s own note in `core/user_tables.py`). + # + # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection + # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the + # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten + # read-through databases (every one of them, which is why the switch was slow) skip the big + # read entirely. + whole = _ut().get(table_key, st=session.runtime) + if whole is None: + # Between the wall and here the table was deleted by another request. Same refusal the + # wall gives, rather than an empty pid set nobody can distinguish from an empty table. + raise err(404, "unknown_table", "that database does not exist") + # ⭐⭐ W36-T21 — AND THE ROW WALL, WHICH IS WHY THIS ARM CAN NO LONGER ALWAYS SKIP THE ROWS. + # + # ⛔ THE PROMISE THIS FUNCTION MAKES IS THAT ITS PID SET IS **IDENTICAL** TO `scoped_pool`'s, + # not merely cheaper. `scoped_pool` now narrows its rows by the permanent filter before taking + # pids, so a set built here from the raw row ids would be WIDER — and every consumer of these + # pids (the workspace envelope, cohort membership, `patch_row`'s scope check) would admit rows + # the read door refuses. That is two ideas of one row set, which is the exact defect class + # `may_open`'s own wave-20 note records ([[one-question-two-normalizers]]). + # + # ⭐ AND W30-T30's SAVING SURVIVES FOR EVERYBODY IT WAS FOR. `row_scope_applies` is False for + # an admin and for every record with no declared filter, which is every account in every + # tenant today — those callers take the id comprehension exactly as before and build no row. + # Only a principal an administrator has actually row-scoped pays the pass, and for them the + # alternative is not "cheaper" but "wrong". + rows = whole.get("rows") or {} + if not perm_scope.row_scope_applies(session.user, table_key): + return frozenset(int(rid) for rid in rows if str(rid).isdigit()), fields_base, whole + keys = {f["key"] for f in fields_base if f.get("key")} + scoped = perm_scope.apply_row_scope( + [{**{k: v for k, v in (row or {}).items() if k in keys}, "pid": int(rid)} + for rid, row in rows.items() if str(rid).isdigit()], + session.user, table_key, fields_base) + return frozenset(r["pid"] for r in scoped), fields_base, whole + + +def ut_write_ctx(session: Session, table_key: str): + """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. + + Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this + or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller + that starts needing rows should fail on an empty list it can see, not on a KeyError. + """ + import aios_grid + from core import grid_events + + limits = [] + # ⭐ W36-T24 / D-214 — ONE lend for the whole pass, so the wall and the grant legs stop + # reading `object_shares` once each. Projected, because this ctx reads no row either. + lent = _ut().lend_defs(session.runtime) + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=lent) + # ⭐⭐ W36-T21 — THE FIELD WALL ON THE **WRITE** CTX, and it is not a copy of the read one. + # `grid_events` refuses a hidden key by asking `ctx.hidden_keys` (`grid_events.py:1802` and + # `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully + # writable on the EVENTS transport — a wall on one wire and not the other is the shape + # `strip_row`'s own note warns about, with the sign flipped. + hidden = _ut_hidden(session, table_key, fields_base, st=lent) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, + admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, + scope_key=table_key, table=_ops(session, table_key, st=lent)) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", + fields_base=fields_base) + # ⭐⭐ W38-T16 — THE SHARED COLUMNS JOIN THE **WRITE** CTX'S CONTRACT TOO, and for the reason + # W36-T21 gave one wall over: `grid_events` refuses a key by asking `ctx.hidden_keys`, so a + # column missing from this list is a column the events transport does not know it must + # refuse. The read door narrowing a name the write door has never heard of is the same + # one-wire wall with the sign flipped. + fields = _ut_shared_fields(session, table_key, fields) + fields, _rows, hidden = _ut_field_wall(session, table_key, fields, [], st=lent) + return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "hidden": hidden, + "derived": aios_grid.cohort_cells(lists), + # ⭐ W37-T12 — the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION + # list for `clean_measure_field`: without it a Metric create on this surface is + # refused `measure_not_offered` even though the read door offers the kind. + # ⛔ THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and + # materialises no rows at all (W30-T30 removed a full table build from the write + # path); computing measure values would put that work straight back. `measure_sets` + # stays empty for the same reason — a write is not a render. + # ⭐ AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN + # ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have + # different jobs: `measure_keys` (from `measures`) is the ADMISSION list + # `clean_filter_tree` uses — non-empty here, so a saved measure CONDITION survives the + # write, which is the failure that comment records for the customer path. Empty + # `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender", + # so the next `/workspace` — which DOES compute the sets, in `ut_assembly` — resolves + # it. Conservative in the right direction: one extra repaint, never a dropped filter. + # ⭐⭐ W38-T19 — THE CAPABILITY IS APPLIED HERE TOO, and this is the site that makes + # the refusal REAL rather than cosmetic: `routes_grid` reads this list as + # `measure_offer`, and `clean_measure_field` fail-closes `measure_not_offered` over + # an empty one. A picker hidden on the read door with this list still full would be + # a control that only stops the honest. + "measures": ut_measures(table_key, session), + "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather + # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. + "limits": limits, "defn": defn} + + +def _ut_hidden(session, table_key, fields, st=None): + """The hidden-field closure for THIS session on THIS database — C1's field half, once. + + ⚠ Named rather than inlined at its four call sites for the reason `may_open`'s own note gives: + four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE + evaluator; this is just the `ut_*` caller's shorthand for it. + + ⭐ W38-T16 — `st` IS THE LEND THE CALLER ALREADY HOLDS, and passing it is free rather than + merely tidy: the closure's field-grant leg resolves against `object_shares`, which is exactly + the bucket `ut_assembly`'s own lend note says the pass is already serving. + """ + import core.perm_scope as perm_scope + return perm_scope.hidden_keys(session.user, table_key, fields, st=st) + + +def _ut_shared_fields(session, table_key, fields): + """`fields` PLUS this database's TENANT-WIDE columns — the read-side merge, W38-T16. + + ⛔⛔ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into + `__shared` for any `ut_*` database since W30-T28, and only the READ-THROUGH door + (`routes_odoo_tables`) ever merged that stratum back. So a shared column written on a + MATERIALISED user table went into the store and was readable by nobody, through any door — + complete, correct, and invisible ([[reachable-is-not-the-same-as-built]]). This is the other + half of that door. + + ⚠ MERGED BEFORE THE WALL, NEVER AFTER, and `routes_odoo_tables` learned this the same way: + the hidden closure must run on the WHOLE contract, or a formula in a shared column that reads + a hidden one sits outside its reach and carries the hidden value out wearing a second name. + ⚠ A key the definition already declares WINS. A shared column is an addition to a database's + contract, never a redefinition of a column that database already has. + """ from core import field_permissions, shared_overlay field_permissions.migrate_legacy_fields( f"{table_key}_table_workspace", st=session.runtime, grant_topic=table_key, shared_key=table_key) defs = shared_overlay.fields(table_key, st=session.runtime) or {} - if not defs: - return fields - have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} + if not defs: + return fields + have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} from core import shares projected = [] for k, f in defs.items(): @@ -757,333 +837,800 @@ def _ut_shared_fields(session, table_key, fields): item["sharedRole"] = role projected.append(item) return list(fields or ()) + projected - - -def _ut_shared_cells(session, table_key, pids): - """`{"": {key: value}}` for the TENANT-WIDE stratum, over THIS session's row set. - - ⛔ `pids` IS THE ROW WALL AND IT IS PASSED, NOT DEFAULTED — `shared_overlay.cells` refuses an - "everything" read by signature for exactly this reason, and the set handed in is the one - `scoped_pool`/`scoped_pids` already narrowed. A cell for a row this session may not see - therefore has nothing to attach to. - """ - from core import shared_overlay - try: - return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime) - except Exception: # noqa: BLE001 - return {} - - -def _ut_field_wall(session, table_key, fields, rows_src, st=None): - """`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires. - - ⭐⭐ W36-T21 — the same three lines `routes_customers.grid_assembly` runs for `customer_data`, - in the same position: AFTER `workspace_wire`, because the closure must cover the user's own - `custom_` and `measure_` columns and those do not exist until it has run. - - ⛔ BOTH WIRES, ALWAYS. `strip_row`'s docstring is the record of why: the field LIST and the - ROW payload are two different wires, and narrowing only the first leaves the value sitting in - the second where anything can read it. A formula (or a rollup) over a hidden column comes out - too — hiding the input while shipping the dependent either leaks the input wearing a derived - column's name or computes a wrong one. - - ⚠ IT TAKES NO `hidden` ARGUMENT, deliberately. The base-level closure the write ctx computed - is a SUBSET of this one by construction — same evaluator, a strictly larger field list — so - accepting it would be a second input that can only ever be redundant, i.e. a parallel code - path with nothing to say ([[one-question-two-normalizers]]). - """ - import core.perm_scope as perm_scope - hide = perm_scope.hidden_keys(session.user, table_key, fields, st=st) - if not hide: - return fields, rows_src, frozenset() - fields = [f for f in fields if f.get("key") not in hide] - rows_src = [perm_scope.strip_row(r, hide) for r in (rows_src or [])] - return fields, rows_src, hide - - -def ut_assembly(session: Session, table_key: str, storage_key: str = "", - consume_corrections: bool = True, with_rows: bool = True, st=None): - """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so - `/workspace` and the events route consume any of the three interchangeably. - - Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is - customer-grain, so there is nothing to offer over user rows. - - ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets - cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its - own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and - wrong the moment the store learned about topics. - - ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the - caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from - `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: - * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over - 963,783 rows nobody was going to look at, which is D-174 in one sentence; - * every materialised `ut_*` database stops allocating a dict per row and sorting them on a - route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per - database switch (owner item 7). - ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers - the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved - on the write door. - """ - import aios_grid - from core import grid_events - - limits = [] - # ⭐⭐ W36-T24 / D-214 — **ONE LEND FOR THE WHOLE PASS**, and it is the shape of the lend that - # keeps D-213's saving. The rows arm needs `rows` off the definition, so it lends the whole - # document; the envelope arm reads no row at all and lends the PROJECTION. Either way the same - # object then serves `object_shares` to the grant legs downstream, so the assembly stops - # reading that bucket once per question asked of it. - if st is None: - st = _ut().lend(session.runtime) if with_rows else _ut().lend_defs(session.runtime) - if with_rows: - # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only - # a READ route passes one — see `scoped_pool`'s own note and contract C5. - pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) - else: - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) - rows_src = [] - - # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it - # as `st`; otherwise this assembly takes its own. Only `user_tables` and `object_shares` are - # served from it, and the assembly writes neither. - ops = _ops(session, table_key, st=st) - # ⭐⭐ W36-T21 — the WRITE half of the field wall. `frozenset()` here meant a column an - # administrator had hidden was still writable through the events transport. - hidden = _ut_hidden(session, table_key, fields_base, st=st) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, - admin=session.admin, fallback_ws=None, seen_ids={}, - # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones - # that happen to carry a scoped `table`. - st=session.runtime, - scope_key=table_key, table=ops) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, - consume_corrections=consume_corrections) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, - storage_key=storage_key, fields_base=fields_base) - # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM, ON A MATERIALISED GRID. Merged HERE, between - # `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two - # lines and for the same reason: the closure must be recomputed on the MERGED contract or a - # formula in a shared column reaches past it. - fields = _ut_shared_fields(session, table_key, fields) - # ⚠ THE CELLS COST A READ, SO THE ENVELOPE ARM DOES NOT PAY IT. `with_rows=False` renders no - # row at all (D-174), and `cells()` over the whole pid set would be a scoped read whose - # result nothing consumes. The COLUMNS still merge above: `/workspace` needs the contract. - shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {} - # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, - # so the closure covers this user's `custom_` and `measure_` columns too. - fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) - if hidden and shared_cells: - # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the - # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak - # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict. - shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden} - for pid, cells in shared_cells.items()} - - # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database - # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real - # lookback catalogue AND the cells to go with it; every other user table still serves `[]`, - # which is the honest answer for a hand-typed one rather than a descope. - today = time.strftime("%Y-%m-%d") - measures = ut_measures(table_key, session) - derived = aios_grid.cohort_cells(lists) # R9: this table's own lists - for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures, - session.runtime.measure_memo).items(): - derived.setdefault(_pid, {}).update(_cells) - - return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, "hidden": hidden, - "derived": derived, - # ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a - # consumer has to test for is a key a consumer forgets to test for, and the consumer - # here (`table_rows`) is layering strata in a fixed order. - "shared_cells": shared_cells, - "measures": measures, - "measure_sets": _ut_measure_sets(table_key, views, measures, pids, today), - "today": today, - # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test - # for is a key a consumer forgets to test for, and this one carries a refusal. - "limits": limits, "defn": defn} - - -def _ut_measure_err(tag, e): - try: - import harness.telemetry as _tel - _tel.error(f"api:{tag}", e) - except Exception: # noqa: BLE001 - pass - - -def ut_measures(table_key, session): - """The lookback-measure OFFER for a user database FOR THIS SESSION, or `[]` — W37-T12 / - owner item 4 (R1), narrowed by the metrics capability in W38-T19. - - ⭐⭐ W38-T19 — `session` IS REQUIRED, NOT DEFAULTED. The function was PRINCIPAL-BLIND, so a - per-user capability had nothing to be consulted by. Defaulting it would leave both existing - call sites compiling and silently ungated, which is the fail-open spelling of the bug rather - than a fix for it — the argument `_module_fields` in `routes_admin` records for its own - `session`. ⛔ AND BOTH CALL SITES MATTER, WHICH IS EASY TO GET HALF RIGHT: `ut_assembly` is - the READ and `ut_write_ctx` is the door `routes_grid` turns into `clean_measure_field`'s - admission set. Gating only the read would take the kind off the picker and leave the CREATE - working for anyone who posts the event. - - ⭐ THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the - topic that DESCRIBES this database. A table with no entity topic, or a topic with no - `measures:` binding, gets `[]` — which is the honest answer for a hand-typed database and is - what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`). - - ⛔ NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model - (`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid, - so a list here would be a second definition of one fact — the argument `_rollup_source_offer` - makes about itself one screen down. - """ - import core.perm_scope as perm_scope - if not perm_scope.may_metrics(session.user, table_key): - return [] - try: - from harness import semantic as sem - topic = sem.topic_for_grid(table_key) - return sem.entity_measures(topic) if topic else [] - except Exception as e: # noqa: BLE001 - _ut_measure_err("ut-measure-offer", e) - return [] - - -def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): - """`{pid: {field key: value}}` for this database's Metric columns. - - ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need - one. `sales_lines.agent` is `rp.agent_id` — a `res.partner` id — and a `ut_odoo_agents` row's - `pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys - `_id` on the partner id). So the group key and the pid are the same integer. The product grid - keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap: - the right answer differs per database and must be read off the binding, never assumed. - """ - import aios_grid - - mfields = aios_grid.measure_fields_of(fields) - if not mfields or not offer: - return {} - from harness import semantic as sem - from harness import windows as _wn - - topic = sem.topic_for_grid(table_key) - by_key = {m["key"]: m for m in offer} - by_window = {} - for f in mfields: - spec = f.get("measure") or {} - if spec.get("key") not in by_key: - continue - rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None - if rng is None: - continue # an unresolvable window is unanswerable — never widen to all time - by_window.setdefault((rng[0], rng[1]), []).append(f) - - out = {} - for (dfrom, dto), group in by_window.items(): - keys = tuple(sorted({f["measure"]["key"] for f in group})) - mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) - if mkey not in memo: - try: - memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, - date_to=dto, team_id=team_id, offer=offer) - except Exception as e: # noqa: BLE001 - _ut_measure_err("ut-measure-column", e) - transient = False - try: - from harness import datastore as _ds - transient = not _ds.ready() - except Exception: # noqa: BLE001 - transient = False - if transient: - continue # a warming store must not memoise as a permanent failure - memo[mkey] = None - vals = memo[mkey] - if vals is None: - continue - # C1's empty-window rule for EVERY row, not only the grouped ones. - filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids} - for f in group: - k, fkey = f["measure"]["key"], f["key"] - for pid, cell in filled.items(): - if k in cell: - out.setdefault(pid, {})[fkey] = cell[k] - if len(memo) > 100: - memo.clear() - return out - - -def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None): - """`{rule id: [pid, …]}` for the measure CONDITIONS in this database's saved views. - - ⛔ SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same - list, so a non-empty `measures` also opens the measure condition in the filter builder — and a - rule id missing from `measureSets` renders PENDING ("Calculating…") and matches nothing, - permanently. Offer and sets are one feature. - """ - if not offer: - return {} - from harness import semantic as sem - - topic = sem.topic_for_grid(table_key) - admitted = {m["key"] for m in offer} - by_id = {} - for v in views or []: - for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted): - rid = str(rule.get("id") or "") - if rid: - by_id[rid] = rule - if not by_id: - return {} - try: - sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id, - keys_by_id=list(pids), offer=offer) - except Exception as e: # noqa: BLE001 - _ut_measure_err("ut-measure-sets", e) - return {} - return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit()) - for rid, ks in sets.items()} - - -def ut_label(defn, key, meta=None): - """THE name of a user table, resolved ONCE (wave 20, item 6a). - - `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that - shows a database name reads through here, because the alternative is what wave 20 found: the - rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker - showed the original (it reads the definition), and neither looked broken. - - ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver - stays because it makes every row already stored — renamed before that fix landed — read - correctly today, without a migration. - """ - return ((meta or {}).get(key, {}).get("name") - or (defn or {}).get("label") or key) - - -def nav_meta(session): - """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" - try: - got = session.runtime.get("nav_meta") - return got if isinstance(got, dict) else {} - except Exception: # noqa: BLE001 - return {} - - + + +def _ut_shared_cells(session, table_key, pids): + """`{"": {key: value}}` for the TENANT-WIDE stratum, over THIS session's row set. + + ⛔ `pids` IS THE ROW WALL AND IT IS PASSED, NOT DEFAULTED — `shared_overlay.cells` refuses an + "everything" read by signature for exactly this reason, and the set handed in is the one + `scoped_pool`/`scoped_pids` already narrowed. A cell for a row this session may not see + therefore has nothing to attach to. + """ + from core import shared_overlay + try: + return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime) + except Exception: # noqa: BLE001 + return {} + + +def _ut_field_wall(session, table_key, fields, rows_src, st=None): + """`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires. + + ⭐⭐ W36-T21 — the same three lines `routes_customers.grid_assembly` runs for `customer_data`, + in the same position: AFTER `workspace_wire`, because the closure must cover the user's own + `custom_` and `measure_` columns and those do not exist until it has run. + + ⛔ BOTH WIRES, ALWAYS. `strip_row`'s docstring is the record of why: the field LIST and the + ROW payload are two different wires, and narrowing only the first leaves the value sitting in + the second where anything can read it. A formula (or a rollup) over a hidden column comes out + too — hiding the input while shipping the dependent either leaks the input wearing a derived + column's name or computes a wrong one. + + ⚠ IT TAKES NO `hidden` ARGUMENT, deliberately. The base-level closure the write ctx computed + is a SUBSET of this one by construction — same evaluator, a strictly larger field list — so + accepting it would be a second input that can only ever be redundant, i.e. a parallel code + path with nothing to say ([[one-question-two-normalizers]]). + """ + import core.perm_scope as perm_scope + hide = perm_scope.hidden_keys(session.user, table_key, fields, st=st) + if not hide: + return fields, rows_src, frozenset() + fields = [f for f in fields if f.get("key") not in hide] + rows_src = [perm_scope.strip_row(r, hide) for r in (rows_src or [])] + return fields, rows_src, hide + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W41-T10 / RULING R3 / CONTRACT C8 — WHERE IS THIS COLUMN USED? +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R3: *"The Field manager row carries owner + who shared it · usage count + permission-rule +# warning · bulk actions."* The row that offers a DELETE has to say what the delete costs, and +# the only number that can say it is one taken over the WHOLE workspace. A count built from the +# caller's own views under-reports by exactly the views the caller cannot see — which is the +# direction that makes a destructive button look safe. +# +#: C8's five members, in C8's order and spelling. Named once so the wire and this file cannot +#: disagree about which five exist. ⛔ ALL FIVE ARE INTS, INCLUDING `permRules` — the lead's dated +#: ruling (2026-08-24) over this ticket's own `done-when`, which says "boolean": C8 is the shape +#: lane C reads, an int carries the boolean (`bool(permRules)`), and a client expecting a number +#: and receiving `true` is [[two-lanes-one-contract-dead-feature]] booked a second time. +_USAGE_KEYS = ("views", "filters", "rollups", "automations", "permRules") + +#: ⚠ THE REFERENCES A VIEW MAKE THAT **BREAK** WHEN THE COLUMN GOES, spelled from the server's own +#: `aios_grid` tuple (`:1679`) rather than from memory: a wrong key name here yields a confident +#: `0`, which is the same lie the absent-key rule below exists to prevent. +_USAGE_DISPLAY_REFS = ("dateField", "stackField", "titleField", + "colorField", "sizeField", "coordField") + +#: `{(tenant, table_key): (token, {field_key: usage})}` — see `_field_usage` for why this is a +#: memo rather than three fresh documents per assembly. Bounded: cleared past the cap rather than +#: grown, the same rule `core.measure_resolve`'s memos keep. +_USAGE_MEMO = {} +_USAGE_MEMO_MAX = 64 + + +def _usage_token(st, ws_key): + """A change token over the THREE buckets `_field_usage` reads, or None if one cannot be asked. + + ⭐⭐ THIS IS WHAT MAKES THE FEATURE AFFORDABLE ON A HOT PATH, and `Store.revision`'s own + docstring is the argument: it *"performs no download, no `get()`, and no copy of the value"* — + the zero-cost property is the feature, not an optimisation. So a repeat assembly over an + unchanged workspace pays three token reads instead of three whole-document deep copies. + ⚠ IT INTRODUCES NO STALENESS THAT THE STORE DOES NOT ALREADY HAVE. The counter is per PROCESS, + and `get()` already caches a bucket for the life of the process and never re-reads it, so a + second container's write is invisible to both. The token is exactly as fresh as the store it + reports on (that same docstring, verbatim). + ⛔ None DISABLES the memo rather than freezing one: an unaskable revision must not pin an + answer forever. + + ⛔⛔ EACH TOKEN ADDRESSES THE DOCUMENT ITS READ ADDRESSES, AND THE ROSTER IS THE ONE WHERE + THOSE DIVERGE. `TenantRuntime.revision` runs the name through `store_key()`, so + `st.revision('users')` watches `users` — while `_field_usage` reads the account + roster through `users.registry()`, which is BARE (see its own note for why: that is the bucket + the wall is built from). On tenant #0 the namespace is empty and the two names coincide, which + is exactly why this would have shipped looking correct: on every OTHER tenant a permission rule + added or removed would bump a revision nobody was watching, and the memo would serve a stale + `permRules` — the member that drives a delete warning — for the life of the process. So the + roster's token comes from the same module-level door its read does. + """ + import core.store as _store + try: + return (str((st.revision(ws_key) or {}).get("token") or ""), + str((st.revision("automations") or {}).get("token") or ""), + str((_store.revision("users") or {}).get("token") or "")) + except Exception: # noqa: BLE001 + return None + + +def _usage_view_refs(cfg): + """`(named, filtered)` for ONE `ViewConfig` — every column it references, and the subset it + FILTERS on. + + ⛔ PRESENTATION MEMBERSHIP IS NOT USE, and refusing it is the whole reason this function + exists. `order`, `visible`, `widths` and `frozenCount` name EVERY column of the database in + EVERY view by construction (`aios_grid` seeds `order` with the full field list), so counting + them would make `views` equal the number of views for every column alike — a number that is + the same for a column nothing touches and for the one three cohorts are built on. What is + counted instead is a reference that BREAKS when the column goes: a filter leaf, a sort, the + group or colour key, and the calendar/kanban/map field picks. + + ⚠ THE FILTER WALK IS `perm_scope._wall_leaf_keys`, NOT A THIRD COPY OF IT. That function + already reaches every `colId` at any depth with groups included, and its own docstring records + what a second walk costs (`routes_admin._leaf_col_ids` is the second, and `verify_api` pins the + two to the same answer). `filter_eval.tree_parts` normalises the `{conj, nodes}` form and the + bare list to one shape, which is the same pair `apply_row_scope` reads. + """ + import core.perm_scope as perm_scope + from harness import filter_eval as fe + if not isinstance(cfg, dict): + return set(), set() + nodes, _conj = fe.tree_parts(cfg.get("filters")) + filtered = {str(k) for k in perm_scope._wall_leaf_keys(nodes)} + named = set(filtered) + for one in (cfg.get("sorts") or ()): + if isinstance(one, dict) and one.get("colId"): + named.add(str(one["colId"])) + for key in ("groupBy", "colorBy"): + if cfg.get(key): + named.add(str(cfg[key])) + display = cfg.get("display") + if isinstance(display, dict): + for ref in _USAGE_DISPLAY_REFS: + if display.get(ref): + named.add(str(display[ref])) + return named, filtered + + +def _usage_field_corpus(defn, ws_doc, fields): + """`{field_key: field}` — every COLUMN DEFINITION this database has, from every stratum. + + ⛔ THREE STRATA, AND MISSING ONE UNDER-REPORTS IN THE DANGEROUS DIRECTION: + * the shared contract (`defn['fields']`), already in hand; + * the merged wire list, which is where the TENANT-WIDE columns arrive — `_ut_shared_fields` + has already merged them from `__shared`, a SEPARATE document from the workspace one + (`shared_overlay`'s residency note: *"its own bucket, beside the per-user one — never a + `__shared__` member"*), and `field_permissions.promote_field` POPS a promoted column out of + its creator's stratum. `perm_scope.user_generated_fields`' own note measured the cost of + forgetting this: 21 of 22 live custom columns are promoted, so a one-document read finds + the one column it is least useful for; + * every OTHER account's `fields` stratum out of the workspace document, which is the half no + caller-scoped list can ever contain and the half the `done-when` is about. + + ⚠ DEDUPED BY KEY, FIRST WRITER WINS, contract first. A key names one column; the same column + forked across strata (which `migrate_legacy_fields` re-promotes on every read) is one column, + and counting each fork would inflate every rollup and automation count by the number of + accounts that have opened the database. + """ + out = {} + for src in (((defn or {}).get("fields") or ()), (fields or ())): + for f in src: + if isinstance(f, dict) and f.get("key") and str(f["key"]) not in out: + out[str(f["key"])] = f + for _uname, ws in (ws_doc or {}).items(): + if not isinstance(ws, dict): + continue + for key, f in ((ws.get("fields") or {}) if isinstance(ws.get("fields"), dict) + else {}).items(): + if isinstance(f, dict) and key and str(key) not in out: + out[str(key)] = f + return out + + +def _usage_column_refs(field): + """`(rollup_keys, automation_keys)` — the columns of THIS table that one column definition + names. + + ⛔ ONLY THE KEYS THAT LIVE ON **THIS** TABLE. `_clean_rollup` stores two kinds and both name + columns, but most of those columns belong to somebody else: on a link rollup, `field`, + `sortBy`, `distinctBy`, `where[].field` and `conditions[].field` are all resolved against the + LINKED table, and only `link` names a column here. On a source-backed rollup the read goes + through a governed topic, so `topic`/`measure`/`groupBy` name the semantic layer and only `on` + names the parent column being matched. Counting the far side would report a column as used by + a rollup that has never read it. + + ⚠ `automation.urlField` IS THE ONLY FIELD-NAMING KEY IN THE BAG — checked against + `aios_grid._clean_automation`, which stores `{kind, source, flowId, urlField, settings}`. + `stageField` looks like a sibling and is not: `automation_engine` stores it as a BOOLEAN marker + (`{"stageField": True}`), so reading it as a key would count the string "True" forever. + """ + rollup, automation = set(), set() + bag = field.get("rollup") if isinstance(field, dict) else None + if isinstance(bag, dict): + src = bag.get("source") + if isinstance(src, dict): + if src.get("on"): + rollup.add(str(src["on"])) + elif bag.get("link"): + rollup.add(str(bag["link"])) + auto = field.get("automation") if isinstance(field, dict) else None + if isinstance(auto, dict) and auto.get("urlField"): + automation.add(str(auto["urlField"])) + return rollup, automation + + +def _field_usage(session, table_key, fields, defn, ops, st=None): + """`{field_key: {views, filters, rollups, automations, permRules}}` — C8's `usage`, computed + over the WHOLE workspace, or `{}` when it cannot be established. + + ⭐⭐ W41-T10 / R3 / C8. The `done-when`'s load-bearing clause is *"over the whole workspace, + not from a partial client list"*: the assembly's own `ws` holds this caller's stratum merged + with the views shared TO them, so a count taken from it would miss every private view every + other account owns — and the consumer is a delete button. + + ⛔ ABSENT MEANS "NOT BUILT YET", NEVER "ZERO", which is C8's stated polarity and the reason + this returns `{}` rather than zeros on a failed read. The rule it applies is + `perm_scope.user_generated_fields`' — *"`None` IS NOT `[]`"* — one door over: `[]` resolves to + "nothing references this column", and a confident `0` on a read that did not happen is + rendered by a UI as *safe to delete*. Two legs can be unresolvable and both refuse: + * the workspace document, if the store will not open it; + * the account roster, if it comes back EMPTY. A live tenant always has at least its own + administrator, so an empty roster is a failed read wearing the shape of an answer. + An empty `automations` bucket is NOT such a case: a tenant with no automations is an ordinary + tenant, and zero is the true count. + + ⛔ THE ROSTER IS READ **BARE**, THROUGH `users.registry()`, AND THAT IS DELIBERATE. Permission + records live on the user record, and `deps._user_for` and `routes_admin._registry` both build + the wall from the un-namespaced `users` bucket. `session.runtime.get('users')` would namespace + the key, find nothing on every tenant but #0, and report `permRules: 0` for all of them — + the exact confident zero the paragraph above refuses. `_tenant_of` is IMPORTED rather than + restated, and both sides of the comparison go through it, so the pre-wave default cannot be + spelled two ways ([[two-orgs-one-display-name]]); it is the same comparison + `routes_admin.get_perms` makes. + + ⛔ O(1) DOCUMENT READS, NEVER O(FIELDS). D-214's whole history is on this route and the wall + beside it: `visible_fields -> hidden_keys -> field_grant_hidden` reaching `core.shares` PER KEY + is the shape two tickets were spent removing. Three documents are opened for the whole field + list — the workspace document (every stratum), the tenant's `automations` bucket and the + account roster — and nothing here reads anything per column. The shared-column definitions and + the shared contract cost NOTHING: `_ut_shared_fields` and `scoped_pool` have already read them + and hand them in. `_usage_token` then collapses the repeat case to three zero-cost tokens, + which is what keeps `/notifications` (`routes_alerts` calls this assembly once per alert) from + paying for the field manager. + + ⚠ IT DISCLOSES THAT A COLUMN IS USED IN VIEWS THIS VIEWER CANNOT OPEN, and that is the + ticket's instruction rather than an oversight. Only an integer crosses: no view name, no + owner, no filter value. The alternative is the caller-scoped count the `done-when` forbids. + """ + keys = {str(f["key"]) for f in (fields or ()) + if isinstance(f, dict) and f.get("key")} + if not keys: + return {} + lent = st if st is not None else session.runtime + memo_key = (str(getattr(session, "tenant", "") or ""), str(table_key)) + token = _usage_token(lent, ops.table_key) + if token is not None: + hit = _USAGE_MEMO.get(memo_key) + if hit and hit[0] == token: + return hit[1] + try: + # ⚠ `ops.st` and `ops.table_key`, not a rebuilt literal: the store key `_ops` closed over + # is the one this database's strata actually live under, and a second spelling of the + # suffix is a second thing to keep in step. A lend passes `_table_workspace` straight + # through to the runtime (it serves only `user_tables` and `object_shares`), so this is + # the tenant's own document. + ws_doc = ops.st.get(ops.table_key) or {} + except Exception: # noqa: BLE001 + return {} + try: + from deps import users as _users + from routes_admin import _tenant_of + roster = _users.registry() or {} + except Exception: # noqa: BLE001 + return {} + if not roster: + return {} + try: + flows = lent.get("automations") or {} + except Exception: # noqa: BLE001 + flows = {} + + tally = {k: dict.fromkeys(_USAGE_KEYS, 0) for k in keys} + seen_views = set() + for _uname, ws in (ws_doc or {}).items(): + if not isinstance(ws, dict): + continue + for vid, view in ((ws.get("views") or {}) if isinstance(ws.get("views"), dict) + else {}).items(): + # ⚠ DEDUPED BY VIEW ID ACROSS STRATA. `table_store`'s own rule is ONE HOME PER VIEW — + # a shared view is REMOVED from its creator's stratum — so a second sighting of an id + # is a store that half-moved a view, and counting it twice would report a number + # nobody could reconcile against the rail. + if not isinstance(view, dict) or str(vid) in seen_views: + continue + seen_views.add(str(vid)) + named, filtered = _usage_view_refs(view.get("config")) + for key in named & keys: + tally[key]["views"] += 1 + for key in filtered & keys: + tally[key]["filters"] += 1 + + for _key, field in _usage_field_corpus(defn, ws_doc, fields).items(): + rollup_refs, auto_refs = _usage_column_refs(field) + for key in rollup_refs & keys: + tally[key]["rollups"] += 1 + for key in auto_refs & keys: + tally[key]["automations"] += 1 + + # ⭐ THE FLOW SIDE OF `automations`, and the binding is the product's own, not a new one: + # `(config.targetTable, config.fieldKey)` is what `automation_engine.bind_unbound_fields` + # matches a column to its definition with, and what `disable_for_table` disables a database's + # automations by. A flow bound to a column that is deleted is left writing nowhere, which is + # precisely what the count is asked to disclose. + for _aid, flow in (flows or {}).items(): + if not isinstance(flow, dict): + continue + cfg = flow.get("config") + if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "") != str(table_key): + continue + fkey = str(cfg.get("fieldKey") or "") + if fkey in keys: + tally[fkey]["automations"] += 1 + + # ⚠ ONE COUNT PER PERMISSION RECORD, not per leaf. A "permission rule" is one account's entry + # for this database (`wall_declared` reads exactly this pair of keys to decide whether a + # narrowing is declared at all), so a record naming the column twice is still one rule that + # breaks. This is the member the lead ruled INT: `bool(permRules)` is the warning R3 asks for. + viewer_tenant = _tenant_of(session.user) + for _uname, rec in (roster or {}).items(): + if not isinstance(rec, dict) or _tenant_of(rec) != viewer_tenant: + continue + entry = (rec.get("perms") or {}).get(str(table_key)) + if not isinstance(entry, dict): + continue + # ⚠ THE SAME WALK THE ROW WALL USES. `entry['filter']` is a FilterTree in either of the two + # shapes `filter_eval.tree_parts` reads, which is exactly what `_usage_view_refs` normalises + # for a view — one reader, so the count and the enforcement cannot disagree about which + # columns a rule names. + _named, filtered = _usage_view_refs({"filters": entry.get("filter")}) + filtered |= {str(k) for k in (entry.get("hiddenFields") or ()) if k} + for key in filtered & keys: + tally[key]["permRules"] += 1 + + if token is not None: + if len(_USAGE_MEMO) >= _USAGE_MEMO_MAX: + _USAGE_MEMO.clear() + _USAGE_MEMO[memo_key] = (token, tally) + return tally + + +def ut_assembly(session: Session, table_key: str, storage_key: str = "", + consume_corrections: bool = True, with_rows: bool = True, st=None): + """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so + `/workspace` and the events route consume any of the three interchangeably. + + Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is + customer-grain, so there is nothing to offer over user rows. + + ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets + cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its + own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and + wrong the moment the store learned about topics. + + ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the + caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from + `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: + * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over + 963,783 rows nobody was going to look at, which is D-174 in one sentence; + * every materialised `ut_*` database stops allocating a dict per row and sorting them on a + route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per + database switch (owner item 7). + ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers + the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved + on the write door. + """ + import aios_grid + from core import grid_events + + limits = [] + # ⭐⭐ W36-T24 / D-214 — **ONE LEND FOR THE WHOLE PASS**, and it is the shape of the lend that + # keeps D-213's saving. The rows arm needs `rows` off the definition, so it lends the whole + # document; the envelope arm reads no row at all and lends the PROJECTION. Either way the same + # object then serves `object_shares` to the grant legs downstream, so the assembly stops + # reading that bucket once per question asked of it. + if st is None: + st = _ut().lend(session.runtime) if with_rows else _ut().lend_defs(session.runtime) + if with_rows: + # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only + # a READ route passes one — see `scoped_pool`'s own note and contract C5. + pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) + else: + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) + rows_src = [] + + # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it + # as `st`; otherwise this assembly takes its own. Only `user_tables` and `object_shares` are + # served from it, and the assembly writes neither. + ops = _ops(session, table_key, st=st) + # ⭐⭐ W36-T21 — the WRITE half of the field wall. `frozenset()` here meant a column an + # administrator had hidden was still writable through the events transport. + hidden = _ut_hidden(session, table_key, fields_base, st=st) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, + admin=session.admin, fallback_ws=None, seen_ids={}, + # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones + # that happen to carry a scoped `table`. + st=session.runtime, + scope_key=table_key, table=ops) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, + consume_corrections=consume_corrections) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, + storage_key=storage_key, fields_base=fields_base) + # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM, ON A MATERIALISED GRID. Merged HERE, between + # `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two + # lines and for the same reason: the closure must be recomputed on the MERGED contract or a + # formula in a shared column reaches past it. + fields = _ut_shared_fields(session, table_key, fields) + # ⭐⭐ W41-T10 / RULING R3 / CONTRACT C8 — **`usage`, STAMPED ONCE, HERE.** + # + # ⛔ IN THE ASSEMBLY AND ON BOTH ARMS, WHICH IS NOT THE CHEAP CHOICE AND IS THE ONLY CORRECT + # ONE. The obvious saving is to build it only when `with_rows is False`, i.e. only for + # `/workspace`. MEASURED IN THE CLIENT INSTEAD OF ASSUMED: `useCustomerData.withWorkspace` + # takes an `adoptFields` flag, and the LOAD path passes it **False** (`useCustomerData.ts:479`) + # because on first paint the ROWS payload's contract is the newer one — its own comment is the + # record of a saved route that never appeared when that precedence was the other way round. So + # a `usage` present only on `/workspace` would be overwritten by the rows payload on every + # first paint and appear only after a write. That is [[two-lanes-one-contract-dead-feature]] + # with both halves correct, which this wave has already booked once. + # + # ⚠ AFTER THE MERGE AND BEFORE THE WALL, W41-T01's position on `customer_data` and for its two + # reasons. After, or the tenant-wide columns are not in the list to stamp and are not in the + # corpus to count FROM. Before, so `_ut_field_wall` deletes a column's usage bag along with the + # column: a reader who may not see a column never learns how much of the workspace depends + # on it. + # + # ⛔ A NEW DICT PER COLUMN, NEVER AN IN-PLACE STAMP — `_ut_shared_fields` returns + # `dict(f, source=…)` copies for the shared stratum but passes the rest through by reference, + # and `_usage_field_corpus` hands those same objects back. Mutating one would write a count + # into a definition the next reader shares. + # ⛔ AND ONLY `usage`. `class` shipped with W41-T01, `agg` belongs to lanes B and D and + # `descriptionEdited` to W41-T11; C8 is explicit that an absent key means "not built yet". + # + # ⚠ LENIENT, LIKE EVERY OTHER DISPLAY READ ON THIS ASSEMBLY, and C8's absent-key polarity is + # what makes degrading safe: a client renders nothing for a missing `usage` rather than a + # wrong "0 uses" beside a delete button, so a store that will not open costs the counts and + # not the grid. + # ⚠ A COPY OF THE BAG, because `_field_usage` MEMOISES its answer: handing the memo's own dict + # out would let anything downstream write into the number the next request reads. + try: + _usage = _field_usage(session, table_key, fields, defn, ops, st=st) + except Exception: # noqa: BLE001 + _usage = {} + if _usage: + fields = [dict(f, usage=dict(_usage[f["key"]])) + if isinstance(f, dict) and _usage.get(f.get("key")) else f + for f in fields] + # ⭐⭐ C8's `descriptionEdited`, AND IT CLOSES A CONSUMER THAT WAS ALREADY WRITTEN. + # W41-T11 built the producer (`user_tables.description_edited`, R19's custody mark scoped to the + # description) and correctly stopped at its own fence, which is one core file. Measured across + # the lane branches afterwards: `filter-kit/fieldClass.ts::descriptionEditedOf` ALREADY READS + # THIS KEY and nothing sent it, so the control it feeds would have rendered nothing forever with + # every gate on both sides green. That is [[two-lanes-one-contract-dead-feature]] with the halves + # inverted -- a consumer waiting on a producer, which the wire-shape design permits in either + # order but only if somebody checks. + # ⚠ THE SAME POSITION AND THE SAME POLARITY AS `usage` ABOVE: after the shared merge, before the + # field wall, a new dict per column, and lenient. C8 is explicit that an ABSENT key means "not + # built yet" rather than `false`, and `descriptionEditedOf` reads it strictly for that reason -- + # so a column this cannot resolve must carry NO key rather than a confident `False`, which a + # reader would render as "the shipped default is still in force". + try: + _de = _ut() + fields = [dict(f, descriptionEdited=_de.description_edited(f)) + if isinstance(f, dict) else f + for f in fields] + except Exception: # noqa: BLE001 + pass + # ⚠ THE CELLS COST A READ, SO THE ENVELOPE ARM DOES NOT PAY IT. `with_rows=False` renders no + # row at all (D-174), and `cells()` over the whole pid set would be a scoped read whose + # result nothing consumes. The COLUMNS still merge above: `/workspace` needs the contract. + shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {} + # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, + # so the closure covers this user's `custom_` and `measure_` columns too. + fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) + if hidden and shared_cells: + # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the + # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak + # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict. + shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden} + for pid, cells in shared_cells.items()} + + # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database + # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real + # lookback catalogue AND the cells to go with it; every other user table still serves `[]`, + # which is the honest answer for a hand-typed one rather than a descope. + today = time.strftime("%Y-%m-%d") + measures = ut_measures(table_key, session) + derived = aios_grid.cohort_cells(lists) # R9: this table's own lists + for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures, + session.runtime.measure_memo).items(): + derived.setdefault(_pid, {}).update(_cells) + + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "hidden": hidden, + "derived": derived, + # ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a + # consumer has to test for is a key a consumer forgets to test for, and the consumer + # here (`table_rows`) is layering strata in a fixed order. + "shared_cells": shared_cells, + "measures": measures, + "measure_sets": _ut_measure_sets(table_key, views, measures, pids, today), + "today": today, + # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test + # for is a key a consumer forgets to test for, and this one carries a refusal. + "limits": limits, "defn": defn} + + +def _ut_measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: # noqa: BLE001 + pass + + +def ut_measures(table_key, session): + """The lookback-measure OFFER for a user database FOR THIS SESSION, or `[]` — W37-T12 / + owner item 4 (R1), narrowed by the metrics capability in W38-T19. + + ⭐⭐ W38-T19 — `session` IS REQUIRED, NOT DEFAULTED. The function was PRINCIPAL-BLIND, so a + per-user capability had nothing to be consulted by. Defaulting it would leave both existing + call sites compiling and silently ungated, which is the fail-open spelling of the bug rather + than a fix for it — the argument `_module_fields` in `routes_admin` records for its own + `session`. ⛔ AND BOTH CALL SITES MATTER, WHICH IS EASY TO GET HALF RIGHT: `ut_assembly` is + the READ and `ut_write_ctx` is the door `routes_grid` turns into `clean_measure_field`'s + admission set. Gating only the read would take the kind off the picker and leave the CREATE + working for anyone who posts the event. + + ⭐ THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the + topic that DESCRIBES this database. A table with no entity topic, or a topic with no + `measures:` binding, gets `[]` — which is the honest answer for a hand-typed database and is + what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`). + + ⛔ NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model + (`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid, + so a list here would be a second definition of one fact — the argument `_rollup_source_offer` + makes about itself one screen down. + """ + import core.perm_scope as perm_scope + if not perm_scope.may_metrics(session.user, table_key): + return [] + try: + from harness import semantic as sem + topic = sem.topic_for_grid(table_key) + return sem.entity_measures(topic) if topic else [] + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-offer", e) + return [] + + +def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): + """`{pid: {field key: value}}` for this database's Metric columns. + + ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need + one. `sales_lines.agent` is `rp.agent_id` — a `res.partner` id — and a `ut_odoo_agents` row's + `pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys + `_id` on the partner id). So the group key and the pid are the same integer. The product grid + keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap: + the right answer differs per database and must be read off the binding, never assumed. + """ + import aios_grid + + mfields = aios_grid.measure_fields_of(fields) + if not mfields or not offer: + return {} + from harness import semantic as sem + from harness import windows as _wn + + topic = sem.topic_for_grid(table_key) + by_key = {m["key"]: m for m in offer} + by_window = {} + for f in mfields: + spec = f.get("measure") or {} + if spec.get("key") not in by_key: + continue + rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None + if rng is None: + continue # an unresolvable window is unanswerable — never widen to all time + by_window.setdefault((rng[0], rng[1]), []).append(f) + + out = {} + for (dfrom, dto), group in by_window.items(): + keys = tuple(sorted({f["measure"]["key"] for f in group})) + mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) + if mkey not in memo: + try: + memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, + date_to=dto, team_id=team_id, offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-column", e) + transient = False + try: + from harness import datastore as _ds + transient = not _ds.ready() + except Exception: # noqa: BLE001 + transient = False + if transient: + continue # a warming store must not memoise as a permanent failure + memo[mkey] = None + vals = memo[mkey] + if vals is None: + continue + # C1's empty-window rule for EVERY row, not only the grouped ones. + filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids} + for f in group: + k, fkey = f["measure"]["key"], f["key"] + for pid, cell in filled.items(): + if k in cell: + out.setdefault(pid, {})[fkey] = cell[k] + if len(memo) > 100: + memo.clear() + return out + + +def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None): + """`{rule id: [pid, …]}` for the measure CONDITIONS in this database's saved views. + + ⛔ SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same + list, so a non-empty `measures` also opens the measure condition in the filter builder — and a + rule id missing from `measureSets` renders PENDING ("Calculating…") and matches nothing, + permanently. Offer and sets are one feature. + """ + if not offer: + return {} + from harness import semantic as sem + + topic = sem.topic_for_grid(table_key) + admitted = {m["key"] for m in offer} + by_id = {} + for v in views or []: + for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted): + rid = str(rule.get("id") or "") + if rid: + by_id[rid] = rule + if not by_id: + return {} + try: + sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id, + keys_by_id=list(pids), offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-sets", e) + return {} + return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit()) + for rid, ks in sets.items()} + + +def ut_label(defn, key, meta=None): + """THE name of a user table, resolved ONCE (wave 20, item 6a). + + `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that + shows a database name reads through here, because the alternative is what wave 20 found: the + rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker + showed the original (it reads the definition), and neither looked broken. + + ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver + stays because it makes every row already stored — renamed before that fix landed — read + correctly today, without a migration. + """ + return ((meta or {}).get(key, {}).get("name") + or (defn or {}).get("label") or key) + + +def nav_meta(session): + """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" + try: + got = session.runtime.get("nav_meta") + return got if isinstance(got, dict) else {} + except Exception: # noqa: BLE001 + return {} + + +def _module_link_targets(session, meta=None, st=None): + """The REGISTRY modules a link column may target, in `brief` shape: `[{key,label,fields}]`. + + ⭐⭐ W41-T15 / RULING R9 / CONTRACT C5 (2026-08-24) — THE WIRE HALF, AND WITHOUT IT THE OTHER + HALF IS UNREACHABLE. `user_tables._clean_link` now accepts `customer_data` and `product_data`, + but the ONLY producer of a link bag is the field editor, whose picker is + `CustomerGrid::fetchLinkTargets` -> `GET /tables?brief=1` -> this route. Until this list + carries them, the two keys are storable and unofferable, which is + [[artifact-with-no-importer]] with the artifact on the server side. + + ⛔ THREE GATES, ALL OF THEM ALREADY THE RESOLVER FOR THEIR OWN QUESTION, because an offer the + caller cannot answer is [[permitted-is-not-answerable]] — a picker listing a source the + feature cannot read spends the user's time and then blanks the column: + + 1. `perms.tenant_has_module` — is this database part of THIS tenant's world at all? Every + provisioned tenant but #0 carries a restricted `modules:` list, so without this a Nurilab + admin (whom both walls below wave through on the break-glass leg) is offered "Odoo + customers" for a connector their workspace does not have. + 2. `perm_scope.nav_may_open` — ADMISSION, and it is the predicate `deps.py`'s module gate + itself asks. `may_read` alone would offer a key the read door then 403s: the two + disagreed once already ([[two-permission-systems-one-armed]]). + 3. `perm_scope.may_read` — the READ wall, the same one every row door here applies. Both, + conjoined, exactly as `routes_admin`'s roster composes them for a surface key. + + ⚠ AND THE FIELD LIST IS THE VIEWER'S, NOT THE SCHEMA'S. `visible_fields` strips the hidden + closure, so a column an administrator hid from this account is not offered as a rollup source + for it. The picker reads `target.fields` to choose which column to fold, so an unstripped list + would name hidden columns in a menu and then fold them into a number. + + ⚠ THE SCHEMA COMES FROM THE STATIC PROVIDERS, the same two arms + `routes_admin._static_field_providers` reads, for the reason that function's own docstring + gives: a second copy of the map is how two surfaces start disagreeing about which columns a + database has. It is restated rather than imported because `routes_admin` is another module's + fence; the SUBJECT (`aios_grid.FIELDS` / `aios_grid.product_fields`) is shared, which is what + stops them drifting. + + ⛔ FAIL-CLOSED ON ANY FAILURE. A store blip that makes a wall unanswerable must drop the offer, + never widen it: a missing row in a picker is a feature that looks absent, an extra one is a + permission decision nobody made. + + ⛔⛔ `st` IS THE CALLER'S LEND AND IT IS NOT AN OPTIMISATION — it is this route's whole reason + for existing in its current shape. `list_tables`' docstring is the record of D-175's third + instance: a tenant with ten databases once paid twenty-one whole-document copies (28.6 MB / + 703 ms warm on tenant #0) to list them, and the fix was to hand the wall the document the + function already holds. `visible_fields` -> `hidden_keys` -> `field_grant_hidden` reaches + `core.shares`, i.e. the `object_shares` bucket, PER KEY — so taking `session.runtime` here + would have quietly added two fresh reads of exactly the kind that ticket removed, to the one + route that must never do it again. `_Lent` serves `user_tables` AND `object_shares`, which is + why the same handle collapses both walls. + + ⚠ `tenant_has_module` deliberately keeps `session.runtime`: it reads `runtime.tenant.config`, + an attribute rather than a document, so a lend buys nothing and forwarding through + `_Lent.__getattr__` for it would only add a hop to reason about. + """ + import aios_grid + import core.perm_scope as perm_scope + import core.perms as perms + import core.registry as registry + ut = _ut() + lent = st if st is not None else session.runtime + providers = {"customer_data": (lambda: aios_grid.FIELDS), + "product_data": aios_grid.product_fields} + out = [] + for key in ut.LINKABLE_MODULE_KEYS: + provider = providers.get(key) + if provider is None: + # A key admitted by `core` that this file has no schema arm for. Not offerable, and + # skipping it silently is right: `LINKABLE_MODULE_KEYS`' own note is the place that + # says adding a key is half a change. + continue + try: + if not perms.tenant_has_module(session.runtime, key): + continue + if not perm_scope.nav_may_open(session.user, key, st=lent): + continue + if not perm_scope.may_read(session.user, key, st=lent): + continue + # ⚠ ONE HANDLE FOR BOTH, which is `visible_fields`' own W38-T16 rule: it RECOMPUTES + # the closure, so a caller that lends to one leg and not the other narrows the field + # list by more than it strips from the rows. + fields = perm_scope.visible_fields( + [f for f in (provider() or ()) if isinstance(f, dict)], + session.user, key, st=lent) + except Exception: # noqa: BLE001 + continue + label = registry.BY_KEY.get(key, {}).get("label") or key + out.append({"key": key, + # `ut_label` so a tenant's `nav_meta` rename wins here exactly as it does on + # the rail. The registry label is the fallback, never a second literal. + "label": ut_label({"label": label}, key, meta), + "fields": [dict(f) for f in fields]}) + return out + + @router.get("/tables") def list_tables(brief: bool = Query(False), session: Session = Depends(require_session)): """This session's user tables — the list the '+ New database' surface renders. - - ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This - route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: - `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on - tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant - with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked - about every table; it is handed the document this function already holds. See + + ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This + route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: + `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on + tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant + with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked + about every table; it is handed the document this function already holds. See `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not available. @@ -1091,6 +1638,16 @@ def list_tables(brief: bool = Query(False), not row counts or record mutability. Reading a full tenant document to decorate that one field editor would move every stored row through Neon for no visible result, so this branch keeps the server-side permission wall but lends its rows-free projection instead. + + ⭐⭐ W41-T15 / R9 / C5 (2026-08-24) — AND `brief=1` NOW OFFERS THE TWO REGISTRY MODULES TOO + (`_module_link_targets`), because `_clean_link` accepts them as link targets and this list is + the only producer of a link bag in the product. + + ⛔ THE BRIEF BRANCH ONLY, and the asymmetry is deliberate. The full listing is the "+ New + database" management surface: it renders row counts, `createdBy` and a delete affordance over + things a tenant OWNS, and `customer_data` is a compiled registry row with its own nav entry + and no such lifecycle. Offering it there would put a database nobody can delete into the list + of databases you delete from. Two callers, two questions, one route. """ ut = _ut() meta = nav_meta(session) @@ -1111,256 +1668,262 @@ def list_tables(brief: bool = Query(False), "created": t.get("created") or "", "rowCount": len(t.get("rows") or {})}) out.append(row) + if brief: + # ⚠ RE-SORTED, not appended. The loop above is already label-ordered; extending it and + # returning would file the two modules after every user table regardless of name, and the + # picker renders this order verbatim. + out.extend(_module_link_targets(session, meta, st=lent)) + out.sort(key=lambda r: (r.get("label") or "").lower()) return {"tables": out} - - -#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a -#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* -#: -#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since -#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product -#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client -#: could make one, so the whole read-through path was reachable by editing Python — the -#: [[artifact-with-no-importer]] shape, twice burned in this repo already. -#: -#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and -#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that -#: topic can group by; a second list here would be a second definition of the same fact, and the -#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding -#: to a metric KEY instead of carrying SQL. -_ROLLUP_CACHE = {} - - -def _rollup_source_offer(): - """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. - - ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE - time refuses silently — the cells are simply left blank, hours later, on a column that looks - configured. Two exclusions do real work: - * a topic with NO dims cannot be grouped at all, so it can never key a parent row; - * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose - inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: - *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only - ever error. - Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what - the user is matching their own column against, and `payment_state` (a value) and - `partner` (an id) are matched to very different columns. - """ - if _ROLLUP_CACHE.get("offer"): - return _ROLLUP_CACHE["offer"] - from harness import semantic as sem - from harness import windows as W - ut = _ut() - - topics, metrics = sem.topics(), sem.metrics() - by_topic = {} - for key, m in metrics.items(): - # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. - if m.get("agg") in ("ratio", "derived"): - continue - by_topic.setdefault(m["topic"], []).append( - {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", - "description": m.get("description") or ""}) - - out = [] - indeterminate = False - for tkey, t in sorted(topics.items()): - dims = ((t.get("store") or {}).get("dims") or {}) - measures = by_topic.get(tkey) or [] - if not dims or not measures: - continue - # ⛔⛔ W37-T13 / /validate-wave 2026-08-19 — THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR - # DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE - # ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one - # door nobody re-read. `semantic._one_binding_offer` — the MEASURE-CATALOG door — refuses - # the same bindings with a stated cause, so the wave shipped **two doors onto one - # vocabulary with only one of them guarded**, and the unguarded one is the picker. - # MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four - # dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised - # `CatalogException: Table with name stock_move does not exist!` — five of the fifty - # (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog - # error, or a column that looks configured and is blank forever - # ([[permitted-is-not-answerable]]). - # ⚠ REUSES `_source_ready` rather than re-deriving readiness: one question, one - # normalizer. `None` (the store could not be asked) refuses TOO — fail closed, exactly as - # the measure door does. - _tbl = (t.get("store") or {}).get("table") - if _tbl: - _rdy = sem._source_ready(_tbl) - if _rdy is not True: - # ⛔ AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment - # records the measured version of this: a cold container with no mirror yet - # refused every binding, the refusal was memoised, and the grid served no measures - # long after the mirror arrived. Here the same shape would freeze a truncated - # PICKER for the life of the process. - indeterminate = indeterminate or (_rdy is None) - continue - out.append({ - "key": tkey, - "label": t.get("label") or tkey, - "grain": t.get("grain") or "", - "dims": [{"key": dkey, - "label": d.get("label") or dkey, - # `store_query` emits `_id` only when the dim carries a display name - # alongside the key; otherwise the value IS the key. `rollup_sql` handles - # both, and the editor says which so the user matches the right column. - "keyedBy": "id" if d.get("name_col") else "value"} - for dkey, d in dims.items()], - "measures": sorted(measures, key=lambda m: m["label"].lower()), - }) - - # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` - # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds - # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the - # validator refuses would let the editor build a field the save door rejects. - windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} - for k in ut.ROLLUP_SOURCE_WINDOWS] - offer = {"topics": out, "windows": windows} - # ⚠ Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped - # because the store could not be asked, this offer is a snapshot of an unknown, not a fact. - if not indeterminate: - _ROLLUP_CACHE["offer"] = offer - return offer - - -@router.get("/tables/rollup-sources") -def rollup_sources(session: Session = Depends(require_session)): - """The topic → metric → dim → window offer the rollup field editor renders. - - ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI - matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below - this line it still resolves; added ABOVE it, this endpoint would silently start arriving as - `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — - this is the cheap ordering that keeps it from mattering. - - ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model - files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live - Odoo data" in the field editor and been able to build a column that can only ever be blank, - because there is no mirror behind it. Three comments (here, in `apiBridge` and on the - `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, - and the mode switch's "render only when there is a choice" guard is built on that promise. - ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already - what `refresh` consults to decide whether these tables may exist at all, and a second copy of - the rule would be a second answer the day a tenant gains a mirror. - """ - import odoo_relational - if not odoo_relational.is_royal(session.tenant): - return {"topics": [], "windows": []} - return _rollup_source_offer() - - -@router.post("/tables", status_code=201) -def create_table(body: dict = Body(default=None), - session: Session = Depends(require_session)): - ut = _ut() - body = body or {} - label = str(body.get("label") or "").strip() - if not label: - raise err(400, "bad_label", "give the database a name") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was created") - source = body.get("source") - try: - key = ut.create(label, session.uname, fields=body.get("fields"), - source=source, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was created") - if not key: - raise err(400, "refused", - f"could not create it. The name may be empty or this tenant already has " - f"{ut.MAX_TABLES} databases") - return {"key": key} - - -@router.patch("/tables/{table_key}") -def patch_table(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Rename a database — IN ITS DEFINITION (wave 20, item 6a). - - ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the - nav's display layer; the definition is what the automation editor's database picker, the - schema drawer and every future reader see. A rename that lands in only one of them leaves a - picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an - amendment for whoever owns that door. - """ - defn = _defn_or_refuse(session, table_key) - ut = _ut() - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can rename it") - label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) - if not label: - raise err(400, "bad_label", "give the database a name") - return {"key": table_key, "label": label} - - -@router.get("/tables/{table_key}/footprint") -def table_footprint(table_key: str, session: Session = Depends(require_session)): - """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). - - Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories - without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled - like the delete itself: only someone who could delete may case the joint. - - ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row - COUNT needs the whole document, and only on a materialised table.""" - defn = _defn_or_refuse(session, table_key, defs_only=True) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - s = session.runtime - views, fields = set(), len(defn.get("fields") or []) - try: - bucket = s.get(f"{table_key}_table_workspace") or {} - for _u, ws in bucket.items(): - if isinstance(ws, dict): - views |= set((ws.get("views") or {}).keys()) - fields += len(ws.get("fields") or {}) # per-user custom/measure strata - except Exception: - pass - import core.shares as shares - g = shares.grants("database", table_key, st=s) - auto = [] - try: - import automation_engine as engine - for aid, d in (engine.all_definitions(s) or {}).items(): - if (d.get("config") or {}).get("targetTable") == str(table_key): - auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) - except Exception: - pass - # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where - # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, - # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the - # projection is not what makes it wrong. - # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 - # in a dialog headed "what dies with this database"). Booked rather than fixed here: this - # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` - # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. - if _ut().materialises(table_key, st=s, defn=defn): - rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) - else: - rows = 0 - return {"rows": rows, "fields": fields, "views": len(views), - "sharedUsers": len(g.get("entries") or []), - "automations": sorted(auto, key=lambda a: a["name"].lower())} - - -@router.delete("/tables/{table_key}") -def delete_table(table_key: str, session: Session = Depends(require_session)): - """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). - - ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the - creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role - grantee could reach this route and delete the database somebody shared with them. The wall - is now the definition's own `createdBy`, the same check the rename route always had. - - Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and - DISABLES bound automations with a status note — never deletes them. The client's confirm - dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the - dialog is a product requirement, not a formality.""" - defn = _defn_or_refuse(session, table_key) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") + + +#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a +#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* +#: +#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since +#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product +#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client +#: could make one, so the whole read-through path was reachable by editing Python — the +#: [[artifact-with-no-importer]] shape, twice burned in this repo already. +#: +#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and +#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that +#: topic can group by; a second list here would be a second definition of the same fact, and the +#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding +#: to a metric KEY instead of carrying SQL. +_ROLLUP_CACHE = {} + + +def _rollup_source_offer(): + """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. + + ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE + time refuses silently — the cells are simply left blank, hours later, on a column that looks + configured. Two exclusions do real work: + * a topic with NO dims cannot be grouped at all, so it can never key a parent row; + * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose + inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: + *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only + ever error. + Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what + the user is matching their own column against, and `payment_state` (a value) and + `partner` (an id) are matched to very different columns. + """ + if _ROLLUP_CACHE.get("offer"): + return _ROLLUP_CACHE["offer"] + from harness import semantic as sem + from harness import windows as W + ut = _ut() + + topics, metrics = sem.topics(), sem.metrics() + by_topic = {} + for key, m in metrics.items(): + # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. + if m.get("agg") in ("ratio", "derived"): + continue + by_topic.setdefault(m["topic"], []).append( + {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", + "description": m.get("description") or ""}) + + out = [] + indeterminate = False + for tkey, t in sorted(topics.items()): + dims = ((t.get("store") or {}).get("dims") or {}) + measures = by_topic.get(tkey) or [] + if not dims or not measures: + continue + # ⛔⛔ W37-T13 / /validate-wave 2026-08-19 — THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR + # DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE + # ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one + # door nobody re-read. `semantic._one_binding_offer` — the MEASURE-CATALOG door — refuses + # the same bindings with a stated cause, so the wave shipped **two doors onto one + # vocabulary with only one of them guarded**, and the unguarded one is the picker. + # MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four + # dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised + # `CatalogException: Table with name stock_move does not exist!` — five of the fifty + # (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog + # error, or a column that looks configured and is blank forever + # ([[permitted-is-not-answerable]]). + # ⚠ REUSES `_source_ready` rather than re-deriving readiness: one question, one + # normalizer. `None` (the store could not be asked) refuses TOO — fail closed, exactly as + # the measure door does. + _tbl = (t.get("store") or {}).get("table") + if _tbl: + _rdy = sem._source_ready(_tbl) + if _rdy is not True: + # ⛔ AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment + # records the measured version of this: a cold container with no mirror yet + # refused every binding, the refusal was memoised, and the grid served no measures + # long after the mirror arrived. Here the same shape would freeze a truncated + # PICKER for the life of the process. + indeterminate = indeterminate or (_rdy is None) + continue + out.append({ + "key": tkey, + "label": t.get("label") or tkey, + "grain": t.get("grain") or "", + "dims": [{"key": dkey, + "label": d.get("label") or dkey, + # `store_query` emits `_id` only when the dim carries a display name + # alongside the key; otherwise the value IS the key. `rollup_sql` handles + # both, and the editor says which so the user matches the right column. + "keyedBy": "id" if d.get("name_col") else "value"} + for dkey, d in dims.items()], + "measures": sorted(measures, key=lambda m: m["label"].lower()), + }) + + # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` + # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds + # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the + # validator refuses would let the editor build a field the save door rejects. + windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} + for k in ut.ROLLUP_SOURCE_WINDOWS] + offer = {"topics": out, "windows": windows} + # ⚠ Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped + # because the store could not be asked, this offer is a snapshot of an unknown, not a fact. + if not indeterminate: + _ROLLUP_CACHE["offer"] = offer + return offer + + +@router.get("/tables/rollup-sources") +def rollup_sources(session: Session = Depends(require_session)): + """The topic → metric → dim → window offer the rollup field editor renders. + + ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI + matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below + this line it still resolves; added ABOVE it, this endpoint would silently start arriving as + `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — + this is the cheap ordering that keeps it from mattering. + + ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model + files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live + Odoo data" in the field editor and been able to build a column that can only ever be blank, + because there is no mirror behind it. Three comments (here, in `apiBridge` and on the + `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, + and the mode switch's "render only when there is a choice" guard is built on that promise. + ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already + what `refresh` consults to decide whether these tables may exist at all, and a second copy of + the rule would be a second answer the day a tenant gains a mirror. + """ + import odoo_relational + if not odoo_relational.is_royal(session.tenant): + return {"topics": [], "windows": []} + return _rollup_source_offer() + + +@router.post("/tables", status_code=201) +def create_table(body: dict = Body(default=None), + session: Session = Depends(require_session)): + ut = _ut() + body = body or {} + label = str(body.get("label") or "").strip() + if not label: + raise err(400, "bad_label", "give the database a name") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was created") + source = body.get("source") + try: + key = ut.create(label, session.uname, fields=body.get("fields"), + source=source, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was created") + if not key: + raise err(400, "refused", + f"could not create it. The name may be empty or this tenant already has " + f"{ut.MAX_TABLES} databases") + return {"key": key} + + +@router.patch("/tables/{table_key}") +def patch_table(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Rename a database — IN ITS DEFINITION (wave 20, item 6a). + + ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the + nav's display layer; the definition is what the automation editor's database picker, the + schema drawer and every future reader see. A rename that lands in only one of them leaves a + picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an + amendment for whoever owns that door. + """ + defn = _defn_or_refuse(session, table_key) + ut = _ut() + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can rename it") + label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) + if not label: + raise err(400, "bad_label", "give the database a name") + return {"key": table_key, "label": label} + + +@router.get("/tables/{table_key}/footprint") +def table_footprint(table_key: str, session: Session = Depends(require_session)): + """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). + + Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories + without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled + like the delete itself: only someone who could delete may case the joint. + + ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row + COUNT needs the whole document, and only on a materialised table.""" + defn = _defn_or_refuse(session, table_key, defs_only=True) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + s = session.runtime + views, fields = set(), len(defn.get("fields") or []) + try: + bucket = s.get(f"{table_key}_table_workspace") or {} + for _u, ws in bucket.items(): + if isinstance(ws, dict): + views |= set((ws.get("views") or {}).keys()) + fields += len(ws.get("fields") or {}) # per-user custom/measure strata + except Exception: + pass + import core.shares as shares + g = shares.grants("database", table_key, st=s) + auto = [] + try: + import automation_engine as engine + for aid, d in (engine.all_definitions(s) or {}).items(): + if (d.get("config") or {}).get("targetTable") == str(table_key): + auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) + except Exception: + pass + # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where + # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, + # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the + # projection is not what makes it wrong. + # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 + # in a dialog headed "what dies with this database"). Booked rather than fixed here: this + # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` + # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. + if _ut().materialises(table_key, st=s, defn=defn): + rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) + else: + rows = 0 + return {"rows": rows, "fields": fields, "views": len(views), + "sharedUsers": len(g.get("entries") or []), + "automations": sorted(auto, key=lambda a: a["name"].lower())} + + +@router.delete("/tables/{table_key}") +def delete_table(table_key: str, session: Session = Depends(require_session)): + """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). + + ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the + creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role + grantee could reach this route and delete the database somebody shared with them. The wall + is now the definition's own `createdBy`, the same check the rename route always had. + + Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and + DISABLES bound automations with a status note — never deletes them. The client's confirm + dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the + dialog is a product requirement, not a formality.""" + defn = _defn_or_refuse(session, table_key) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") try: import automation_control import automation_engine as engine @@ -1376,860 +1939,876 @@ def delete_table(table_key: str, session: Session = Depends(require_session)): raise err(503, "automation_disable_failed", f"the database is still present because its automations could not be disabled: " f"{type(exc).__name__}") - try: - _ut().delete(table_key, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the delete did not land. Try again") - return {"ok": True} - - -#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. -_IG_FORWARDED = set() - - -def _ig_forward(session): - """Bring this tenant's Instagram tables onto the current schema, at most once per process. - - ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE - path. `ut_ensure` calling it is right for a schema an automation is about to append to, and - useless for a change a PERSON is waiting to see: the owner's report was *"the first field is - still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. - The write path stays exactly as it was — this is a second door to the same idempotent call, - not a replacement for it. - - ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per - tenant per process; `migrate_ig_tables` returns without a write when every table is already - current (the common case after the first read); and a failure is SWALLOWED — a migration must - never be the reason a database will not open. - - ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be - retried on every subsequent read of every table for the life of the process — the write path is - still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing - store call on the hot path of a grid that is trying to render. - """ - tenant = str(getattr(session, "tenant", "") or "") - if tenant in _IG_FORWARDED: - return - _IG_FORWARDED.add(tenant) - try: - import automation_engine as engine - engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") - - - -#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized -#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is -#: not — the shape this exists for is one already-paid provider payload per row. -JSON_LIST_MAX = 400 - - -def _thin_json(fields, merged, table_key=""): - """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. - - ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads - `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something - large rather than something empty. `_truncated` is what the viewer keys its fetch on. - - ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the - viewer needs no table key, no record id and no new props threaded down through three - components to find the document — the route that removed the value says where it went. One - writer of that address instead of a server rule and a client rule that must agree forever. - """ - json_keys = [str(f.get("key")) for f in (fields or []) - if str(f.get("type") or "") == "json"] - if not json_keys: - return merged - out = {} - for pid, cells in (merged or {}).items(): - row = cells - for key in json_keys: - raw = cells.get(key) - if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: - if row is cells: - row = dict(cells) - row[key] = json.dumps({ - "_truncated": True, "bytes": len(raw), - "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) - out[pid] = row - return out - - -@router.get("/tables/{table_key}/rows") -def table_rows(table_key: str, session: Session = Depends(require_session)): - """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, - identity}` — so the client's generic topic fetch consumes it with zero new parsing. - - ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell - from the OVERLAY stratum only (that is what makes a custom column render standalone) — a - user table's base values live in its DEFINITION rows, so they are layered UNDER the user's - overlay edits here: base first, overlay wins. Without this every base cell reads empty - (found by this route's own gate check, not by luck).""" - import aios_grid - - _ig_forward(session) - # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. - # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE - # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), - # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask - # about the SAME document in the SAME request. The lend is the fix W31 QA already built for - # `_defn_or_refuse`; this threads it through the three sites that never got it. - # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a - # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the - # value it replaced (contract C5). - _lent = _ut().lend(session.runtime) - g = ut_assembly(session, table_key, st=_lent) - # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge - # this line does not perform since the route was written (owner item 3, 2026-08-09: - # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). - # - # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` - # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table - # is overlay-typed by construction, so it looked up a key that could not be there and every - # such cell rendered blank. - # - # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds - # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed - # in the store exactly as they should. Only the READ-BACK dropped them, which is why the - # value survived the gesture, vanished on reload, and looked like "it reset itself" — and - # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. - # - # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition - # value must not shadow an edit the user has made on top of it. - # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this - # widens what a caller can SEE by exactly their own edits and nothing else. - _overlays = (g.get("ws") or {}).get("overlays") or {} - merged = {} - for _r in g["rows_src"]: - _pid = str(_r["pid"]) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _ov = _overlays.get(_pid) - if isinstance(_ov, dict): - _cells.update(_ov) - merged[_pid] = _cells - # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM GOES ON TOP, and the order is the contract rather - # than a preference. A shared column is one the whole permitted audience must read the SAME - # value in — that is the entire reason `shared_overlay` exists — so a stale per-user value - # left under the same key must not win. `routes_odoo_tables` layers the two in this exact - # order (`overlays.setdefault(pid, {}).update(cells)`) and the two doors must agree. - # ⛔ NEVER `setdefault` ON `merged`: these cells are already narrowed to this session's pids, - # but inventing a row id here would put a row in the payload that `rows_src` never admitted. - for _pid, _cells in (g.get("shared_cells") or {}).items(): - if str(_pid) in merged and isinstance(_cells, dict): - merged[str(_pid)].update(_cells) - # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE - # JSON DOCUMENTS DO NOT RIDE THE LIST. - # - # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% - # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB - # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The - # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole - # vendor response crossed the wire, was parsed by the browser and held in memory purely so a - # clipped first line could be drawn. - # - # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by - # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — - # the one place a person actually reads it, for the one row they opened. The cell that rides - # the list is a VALID small document saying what it stands for, so the preview renders - # honestly instead of showing half a truncated brace. - # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, - # so a tenant using `json` for a short config sees no change at all. - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) - # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ - # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. - _stamp_ai_states(table_key, g["fields"], rows, st=_lent) - # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you - # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this - # database says so here, with its cause and the recommendation, rather than waiting to be - # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the - # honest answer for a table nothing limits — never an absent key, which a client cannot tell - # apart from an older server. - _report = _ut().limit_report(table_key, st=_lent) - # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the - # only thing that ever set this key; the write door never stopped working and every client - # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a - # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the - # customer scope — `grid_events.docs_for` — never a twin here. - from core import grid_events as _ge - return {"fields": g["fields"], "rows": rows, "today": g["today"], - "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, - admin=session.admin, st=session.runtime), - "pulled_at": time.strftime("%Y-%m-%d %H:%M"), - "identity": {"pid": "pid"}, - "scope": {"table": table_key, "rowCount": len(rows)}, - "limits": [_report] if _report else [], - "recordsMutable": _ut().records_mutable(table_key, st=_lent)} - - -#: The per-cell provenance a row carries on the wire, one key per enrichment column. -#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. -#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new -#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces -#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this -#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the -#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off -#: every field key, so no column can ever be called `_ai_*`. -AI_STATE_PREFIX = "_ai_" - - -def _stamp_ai_states(table_key, fields, rows, st=None): - """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. - - ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token - count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest - would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary - is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the - RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. - - ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no - enrichment column is untouched, and a freshly created column adds nothing until something - runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row - instead of against whatever was true when the value was written. - """ - cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] - if not cols: - return rows - import ai_enrich as _ae - for field in cols: - col = str(field.get("key") or "") - marks = _ut().ai_enrich_marks(table_key, col, st=st) - cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} - for row in (rows or []): - if not isinstance(row, dict): - continue - state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) - if state != "empty": - row[AI_STATE_PREFIX + col] = state - return rows - - -@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") -def table_cell(table_key: str, pid: str, fkey: str, - session: Session = Depends(require_session)): - """ONE cell, whole — the other half of `_thin_json`. - - ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor - for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON - viewer opens this for the one row a person is actually reading. - - ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's - view of the table), so this cannot become a side door onto a table the caller may not open — - which is the failure a "just fetch the raw cell" helper invites. - ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must - read back what they typed, not the definition value underneath it. - """ - g = ut_assembly(session, table_key) - field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) - if field is None: - raise err(404, "unknown field", f"{fkey!r} is not a column on this database") - row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) - if row is None: - raise err(404, "unknown record", f"no record {pid!r} in this database") - overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} - value = overlay.get(fkey, row.get(fkey)) - return {"table": table_key, "pid": str(pid), "field": str(fkey), - "value": "" if value is None else str(value)} - - -@router.post("/tables/{table_key}/rows", status_code=201) -def add_row(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). - - ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo - that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the - response rather than assume; the client re-anchors on what came back. - """ - _records_or_refuse(session, table_key) - ut = _ut() - values = (body or {}).get("values") or {} - if not isinstance(values, dict): - raise err(400, "bad_values", "values must be an object of {fieldKey: value}") - try: - rid = ut.add_row(table_key, values, session.uname, st=session.runtime, - rid=(body or {}).get("rid")) - except Exception: - raise err(503, "store_unavailable", "the row was not saved. The store refused") - if rid is None: - # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap - # sentence alone would misdirect — the reader would go and count rows. Ask the same - # validator the law used rather than re-deciding here (one rule, two voices). - pf = ut.profile_field(table_key, st=session.runtime) - if pf and pf["key"] in values: - _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) - if not ok: - raise err(400, "refused", - f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " - f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " - f"profile link (instagram.com/name)") - raise err(400, "refused", - f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") - _refresh_relations(session) - return {"rid": rid, "pid": int(rid)} - - -@router.post("/tables/{table_key}/rows/import", status_code=201) -def import_rows(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. - - Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a - spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, - nothing is matched or overwritten, and the dialog says so before the button is pressed. - - ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate - the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in - an import is not a stray to be tidied away — it means the client offered a target it should - not have, and silently dropping it would leave the user looking for a column of values that - never arrived. The refusal names the column. - - ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every - profile cell validates, because a half-imported file is the worst outcome available: the user - cannot tell which rows landed without reconciling the spreadsheet by hand. - """ - _records_or_refuse(session, table_key) - ut = _ut() - rows_in = (body or {}).get("rows") - if not isinstance(rows_in, list) or not rows_in: - raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") - if any(not isinstance(r, dict) for r in rows_in): - raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") - defn = _defn_or_refuse(session, table_key) - by_key = {f["key"]: f for f in (defn.get("fields") or [])} - asked = {k for r in rows_in for k in r} - unknown = sorted(k for k in asked if k not in by_key) - if unknown: - raise err(400, "unknown_field", - f"this database has no column {unknown[0]!r}") - computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) - if computed: - label = by_key[computed[0]].get("label") or computed[0] - raise err(400, "computed_field", - f"{label!r} is worked out from other columns, so it cannot be imported into") - # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. - # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a - # second client, or a future importer met no wall at all and the string landed verbatim in a - # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE - # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every - # rule on this route exists to prevent. The sentence names the row and the column, because - # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. - for index, row in enumerate(rows_in): - for key, value in row.items(): - why = ut.cell_type_refusal(by_key[key], value) - if why: - raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") - try: - made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "nothing was imported. The store refused") - if made is None: - raise err(400, "refused", - f"nothing was imported. {len(rows_in)} rows would take this database past " - f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") - _refresh_relations(session) - return {"imported": len(made), "pids": [int(r) for r in made]} - - -# --------------------------------------------------------------------------------------------- -# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) -# --------------------------------------------------------------------------------------------- -# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same -# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's -# definition to everyone without handing over the table. This supersedes wave 17's "fields are -# per-user" law for this path only; the connector scopes keep their own model. -# -# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the -# per-user workspace overlay, which is why the automation editor's "Automation column" picker -# could not see a column the user had just created — it reads the DEFINITION. Same defect shape -# as the rename (item 6a): two places to look, and the surfaces disagreed silently. - -def _field_or_refuse(session, table_key, fkey=""): - """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. - - ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and - `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they - write through the `user_tables` write doors), so no projected snapshot survives into a write. - ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 - draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" - defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) - ut = _ut() - if not ut.is_user_table(table_key, st=session.runtime): - raise err(400, "not_a_user_table", - "only a user-created database has an editable schema. A connected source " - "owns its own columns") - # ⭐⭐ W36-T21 — A HIDDEN COLUMN IS NOT EDITABLE, AND THIS IS THE DOOR THAT HAD TO SAY SO. - # `EventCtx.hidden_keys` walls the events transport; the REST schema routes (rename, retype, - # delete a column) do not pass through it at all. Without this an account that could not SEE - # `unit_cost` could still DELETE it for the whole tenant — the loudest possible version of a - # wall that exists on one wire only. ⚠ Read the closure off the DECLARED fields, which is what - # `routes_admin` validates a stored `hiddenFields` list against. - if fkey and str(fkey) in _ut_hidden(session, table_key, defn.get("fields") or []): - raise err(403, "forbidden", - "this database has no column by that name that you may edit") - if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, - st=session.runtime): - field = next((f for f in (defn.get("fields") or []) - if f.get("key") == str(fkey)), None) - if isinstance((field or {}).get("automation"), dict) \ - and field["automation"].get("preset") is True: - # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not - # ut.may_edit_field` above is the wall). Said out loud because the sentence itself - # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a - # blanket "pre-set fields are locked" would now be the server explaining a refusal - # it did not make — `preset_editable` is the one predicate that answers this. - raise err(403, "preset_field_locked", - "this is a pre-set column, so its name and type are fixed; you may sort, " - "filter or hide it, edit any Rollup column, and add your own columns") - raise err(403, "forbidden", "that column can only be changed by the database's creator " - "or an admin") - if not fkey and not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can add a column") - return defn - - -@router.post("/tables/{table_key}/fields", status_code=201) -def add_field(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - _field_or_refuse(session, table_key) - ut = _ut() - field = ut.add_field(table_key, body or {}, st=session.runtime) - if not field: - # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers - # None for every refusal, so this route said "check the name and type" to somebody whose - # name and type were fine and whose automation column named a flow that does not exist. - # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the - # one thing that was never wrong. Checked HERE, in the route's own words, because the - # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never - # re-implements it (a second copy of the rule is how two doors start disagreeing). - raise err(400, "refused", - _refusal_sentence(ut, session, body or {}, table_key=table_key)) - if field.get("type") == "link": - synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) - field = synced.get("field") or field - # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route - # became reachable for one. It was gated on `link` alone, while `patch_field` and - # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first - # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). - # The user would have created the column, watched a 201 come back, and read a blank cell for - # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. - # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in - # the tenant, and adding a text column has nothing to fold. The condition is now "is this - # field relational", which is the question that was always meant. - if field.get("type") in ("link", "rollup"): - _refresh_relations(session) - return _with_dropped(ut, {"field": field}, body) - - -def _with_dropped(ut, out, body): - """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). - - ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag - cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and - `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the - defect. T51's contract is that an unknown config key is dropped **and named**, so the naming - rides the response beside the accepted field rather than inside the validator. - - ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to - ignore the key, which is how a report stops being read before it stops being true. - """ - dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) - if dropped: - out = dict(out) - out["dropped"] = dropped - return out - - -def _refusal_sentence(ut, session, body, table_key="", fkey=""): - """Why was this column refused? The specific reason when we can name one, the general list - otherwise — never a specific-sounding guess.""" - # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag - # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment - # column would otherwise be explained by the flow law -- "pick a flow, or make this an - # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user - # never touched. - if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": - bag = (body or {}).get("aiEnrich") - if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): - return ("an AI enrichment column needs a prompt. It is the only thing that can " - "produce a value here, so a column without one would stay empty forever") - # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is - # what the contract asks for and what makes it actionable — "at most one" sends the reader - # hunting through a 40-column schema for a flag they cannot see from the header. - if isinstance((body or {}).get("profile"), dict): - # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at - # once (an `int` profile column on a table that already has a profile column), and the - # TYPE is the one that is wrong about what the caller just sent — unconditionally, no - # matter what else is on the table. Answering "you already have one" to somebody whose - # real mistake was the column type sends them to fix the wrong thing, which is the - # misdirection D-46 closed one door over. - if str((body or {}).get("type") or "text").strip().lower() != "text": - return ("a profile column is a flag on an ordinary TEXT column. It validates what " - "is typed into it, which it can only do for text") - existing = ut.profile_field(table_key, st=session.runtime) if table_key else None - if existing and existing.get("key") != str(fkey): - return (f"this database already has a profile column: " - f"{existing.get('label') or existing.get('key')!r}. A database has at most " - f"one, so the automation knows which handle to enrich; edit that column, or " - f"take the flag off it first") - bag = (body or {}).get("automation") - if isinstance(bag, dict): - flow = str(bag.get("flowId") or "").strip() - if not flow: - return ("an automation column has to name the automation that fills it. Pick a " - "flow, or make this an ordinary column") - if not ut.flow_bound(bag, st=session.runtime): - return (f"this column names automation {flow!r}, which does not exist in this " - f"workspace. It may have been deleted; pick a flow that is still there") - kind = str((body or {}).get("type") or "").strip() - if kind and kind not in ut.UT_FIELD_TYPES: - return (f"{kind!r} is not a column type here (types: " - f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") - return (f"the column was refused. Check the name and type, or the table may be at its " - f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") - - -@router.patch("/tables/{table_key}/fields/{fkey}") -def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Edit one column's definition, and MIGRATE its values when options are renamed. - - ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: - [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed - Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and - every saved view that filtered on it. - """ - _field_or_refuse(session, table_key, fkey) - ut = _ut() - body = body or {} - migrated = None - renames = body.get("renames") - if renames: - try: - migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the rename did not land. Try again") - # The per-user workspace strata and any view filter naming the old value are the OTHER - # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an - # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a - # missing counterpart must not lose the half that DID land. - try: - import core.table_store as table_store - fn = getattr(table_store, "rename_choice_values", None) - if callable(fn): - fn(table_key, fkey, renames, st=session.runtime) - migrated = dict(migrated or {}, workspace=True) - except Exception: # noqa: BLE001 - migrated = dict(migrated or {}, workspace=False) - field = ut.patch_field(table_key, fkey, body, st=session.runtime) - if not field: - # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as - # adding one, so it must get the same sentence naming the column that already holds the - # flag — a patch that answered "check the name and type" would send the reader to the - # one thing that was never wrong (the D-46 lesson, one door over). - raise err(400, "refused", - _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) - synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) - field = synced.get("field") or field - _refresh_relations(session) - out = {"field": field} - if migrated is not None: - out["migrated"] = migrated - return _with_dropped(ut, out, body) - - -def _fire_on_change(table_key, pid, changed, session): - """Run any `on_change` enrichment column whose prompt names a cell that just moved. - - ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. - `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a - 20,000-row import into 20,000 whole-document reads on the single process this product runs - (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and - takes the definition, so this reads once and asks about every column. - - ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; - an enrichment that could not run is a missing value, not a lost edit, and the run's own report - carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a - fan-out here would put a vendor call on the critical path of every keystroke-commit. - """ - import ai_enrich as _ae - - try: - defn = _ut().get(table_key, st=session.runtime) or {} - wanted = _ae.on_change_fields(defn, changed.keys()) - for field in wanted: - # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still - # somebody's edit spending somebody's tokens, so it is booked against the person who - # typed rather than left unattributed. - _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], - user=session.uname) - except Exception: # noqa: BLE001 - pass - - -@router.post("/tables/{table_key}/fields/{fkey}/enrich") -def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. - - ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides - (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the - tenant by opening a grid. - - `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the - value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic - plan, where the policy and the never-overwrite-a-human law both apply. The two are one - function with one flag, not two runners. - - ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by - reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a - ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a - populated `skipped` is a correct, informative answer, and the client must render it rather - than treat it as success. - """ - _field_or_refuse(session, table_key, fkey) - import ai_enrich as _ae - - rows = (body or {}).get("rows") - if rows is not None and not isinstance(rows, list): - raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " - "rows this column's own settings choose") - # The caller's permitted pool, the same one the row doors use. A named row outside it is - # dropped rather than refused: a stale client naming a record that has been deleted or moved - # out of scope should not fail a run over the rows it can legitimately fill. - if rows is not None: - allowed = {str(p) for p in scoped_pids(session, table_key)[0]} - rows = [str(r) for r in rows if str(r) in allowed] - # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" - # sends `always`. Anything else falls back to the column's own saved policy rather than to a - # default, so a typo cannot quietly widen what a run touches. - report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, - policy=str((body or {}).get("scope") or "") or None, - # A named row set through THIS door is a person asking. - manual=rows is not None, - # ⭐ W35-T41 / C7 — the usage ledger's attribution. - user=session.uname) - if report.get("problem"): - # A run that could not start at all is not a 200: nothing was attempted, nothing was - # spent, and the reason is actionable (no provider configured, or the wrong column). - raise err(400, "enrich_refused", str(report["problem"])) - return report - - -@router.delete("/tables/{table_key}/fields/{fkey}") -def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): - _field_or_refuse(session, table_key, fkey) - if not _ut().delete_field(table_key, fkey, st=session.runtime): - raise err(400, "refused", - "that column could not be removed. A database must keep at least one") - _refresh_relations(session) - return {"deleted": fkey} - - -@router.delete("/tables/{table_key}/rows/{rid}") -def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): - # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. - # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* - # A single DELETE was FIVE whole-document deep copies before the commit even began — three in - # the guard (`get` + `may_open` + `records_mutable`) and two more inside - # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads - # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** - # (D-185), so the guard alone was seconds of copying to answer questions about one row. - # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` - # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document - # under the store lock, so a lent snapshot can never be the thing written back. - # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually - # consistent delete is indistinguishable from one that did not work. This makes the guard - # cheap; it does not make the commit optimistic. - lent = _ut().lend(session.runtime) - _records_or_refuse(session, table_key, st=lent) - # ⭐⭐ W36-T21 — THE ROW SCOPE, ON THE DELETE DOOR. `patch_row` has asked this since it was - # written (`pid not in g["pids"]` -> 403) and this door never did, because until now every - # account that could open a `ut_*` database could see every row of it. The moment an - # administrator can row-scope one, "may not SEE row 5" and "may DELETE row 5" become two - # different answers unless this is here — and delete is the one that cannot be undone. - # ⚠ Guarded on `row_scope_applies` so an unscoped account pays nothing: for them the pid set - # is every row and the question has one answer. - import core.perm_scope as _ps - if _ps.row_scope_applies(session.user, table_key): - _pids, _f, _d = scoped_pids(session, table_key) - if not str(rid).isdigit() or int(rid) not in _pids: - raise err(403, "out_of_scope", "that row is not in this database") - try: - ok = _ut().delete_row(table_key, rid, st=lent) - except Exception: - raise err(503, "store_unavailable", "the delete did not land. Try again") - if not ok: - raise err(400, "refused", "rows can only be deleted from user-created databases") - _refresh_relations(session) - return {"ok": True} - - -@router.patch("/tables/{table_key}/rows/{pid}") -def patch_row(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Cell edits — the products PATCH on the user-table ctx. Routed through - `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; - the accepted values are read BACK from the bucket, never echoed from the request.""" - from core import grid_events - - updates = dict(body or {}) - if not updates: - raise err(400, "empty_patch", "no fields to update") - _records_or_refuse(session, table_key) - g = ut_assembly(session, table_key, consume_corrections=False) - if pid not in g["pids"]: - raise err(403, "out_of_scope", "that row is not in this database") - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], - admin=session.admin, fallback_ws=None, seen_ids={}, - # ⭐⭐ W36-T21 — the assembly's OWN closure, not an empty set. `g["fields"]` is already - # stripped, but `grid_events` asks `ctx.hidden_keys` by name: a PATCH naming a hidden - # column would otherwise be accepted on a payload that never showed it. - hidden_keys=g.get("hidden") or frozenset(), - st=session.runtime, # R6b (D-16) - scope_key=table_key, table=_ops(session, table_key)) - try: - grid_events.handle_one( - {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", - "pid": pid, "updates": updates}, ctx) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable. Your change was not saved") - # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. - # - # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH - # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. - # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's - # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began - # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` - # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring - # claiming otherwise is what makes the next reader look for a merge bug that is not here. - # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY - # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` - # asks whether THIS write landed, and this write goes to the definition. - stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) - .get(str(pid)) or {}) - accepted = {k: stored.get(k) for k in updates if k in stored} - - def _took(k): - """Did the cell TAKE this write? Normally that is "stored == asked". - - ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` - and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell - the client to roll back a write that landed. But it cannot simply be exempted either — - a junk handle leaves the OLD value sitting in `stored`, which would then read as - accepted. So the question asked is the exact one: **is what is stored the canonical form - of what was asked?** Anything else is a genuine refusal. - """ - if k not in accepted: - return False - want = str(updates[k]) - if stored.get(k) == want: - return True - pf = _ut().profile_field(table_key, st=session.runtime) - if pf and pf["key"] == k: - handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) - return bool(ok) and stored.get(k) == handle - return False - - refused = sorted(k for k in updates if not _took(k)) - # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write - # this cell?" is not recoverable from the value afterwards, so the only place to record it is - # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic - # run overwrite it, whatever the column's `overwrite` policy says. - # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused - # write `human` would freeze a cell against the agent on the strength of an edit that never - # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. - took = {k: v for k, v in accepted.items() if k not in refused} - if took: - try: - _ut().note_human_edit(table_key, took, pid, st=session.runtime) - except Exception: # noqa: BLE001 - # Provenance is metadata about a write that has already succeeded. Failing the - # request here would tell the user their edit was lost when it was not. - pass - _fire_on_change(table_key, pid, took, session) - out = {"pid": pid, "updates": accepted} - if refused: - out["refused"] = refused - # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a - # profile blank cleared. Without this the grid keeps painting a stale follower count under - # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other - # half: the client may keep only what the server actually took, and must be TOLD what else - # moved. Derived by diffing this row against what was asked for, so it cannot drift from - # whatever the clear rule decides to touch. - also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} - cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) - if cleared: - out["cleared"] = cleared - # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, - # amendment A5; SESSION B built and gated the function and correctly declared it INERT until - # this line existed, citing [[flag-shipped-without-its-writer]]). - # - # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead - # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the - # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we - # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try - # it again, the account is back". Without this call that person has no way back at all, and - # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. - # - # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this - # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a - # human touched the handle cell. - # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- - # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must - # be readable in the log rather than degrade into "the re-arm quietly stopped working". - _pf = _ut().profile_field(table_key, st=session.runtime) - if _pf and _pf["key"] in updates: - try: - import automation_engine as _engine - _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") - _refresh_relations(session) - return out - - -# ── CONTRACT C1 (W36-T20): THE MIRROR READER ────────────────────────────────────────────────── -# ⭐⭐ R6. `core.perm_scope.scoped_table` is the ONE door to any database's rows and it answers for -# a MATERIALISED `ut_*` table entirely on its own — deliberately, so a cold process (E's sandbox -# subprocess, a worker, a gate) that never imported a route still gets the right answer. A -# READ-THROUGH grid is the one arm it cannot serve alone: those ten databases store no rows in the -# tenant document at all, and their rows live in the DuckDB mirror behind `routes_odoo_tables`, -# two layers above `core`. -# -# ⛔ ONE FETCH, NOT A SECOND ONE. This hands C1 the SAME `_read_through_rows` that `scoped_pool` -# and `scoped_pids` already share, so the rows a script sees through C1 are byte-for-byte the rows -# the grid sees — by construction, not by a second query that agrees today (W31-T20's argument, one -# caller further out). -# -# ⚠ AND `TooBigToMaterialise` BECOMES A REPORTED REFUSAL, NEVER A SHORT ANSWER. Standing rule 1's -# second sentence, and the shape is `_PID_SCOPE_LIMIT`'s so nothing downstream needs a second -# vocabulary for it. -def _c1_mirror_rows(table_key, field_keys, st): - """C1's mirror arm: `perm_scope.register_mirror`'s reader over `_read_through_rows`.""" - import core.perm_scope as perm_scope - - try: - return _read_through_rows(table_key, field_keys, rt=st) - except _too_big() as e: - raise perm_scope.Unresolvable( - cause=str(e), - recommendation=_PID_SCOPE_LIMIT["recommendation"], - subject="rows", effect="unresolved") from e - except RuntimeError as e: - raise perm_scope.Unresolvable( - subject="rows", effect="unreadable", cause=str(e), - recommendation="the connector mirror is not ready on this process; retry once the " - "store has finished opening") from e - - -def _register_mirror(): - """Declare the mirror reader to C1. Called at import; returns True once it is registered.""" - import core.perm_scope as perm_scope - return perm_scope.register_mirror(_c1_mirror_rows) - - -_C1_MIRROR = _register_mirror() + try: + _ut().delete(table_key, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the delete did not land. Try again") + return {"ok": True} + + +#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. +_IG_FORWARDED = set() + + +def _ig_forward(session): + """Bring this tenant's Instagram tables onto the current schema, at most once per process. + + ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE + path. `ut_ensure` calling it is right for a schema an automation is about to append to, and + useless for a change a PERSON is waiting to see: the owner's report was *"the first field is + still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. + The write path stays exactly as it was — this is a second door to the same idempotent call, + not a replacement for it. + + ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per + tenant per process; `migrate_ig_tables` returns without a write when every table is already + current (the common case after the first read); and a failure is SWALLOWED — a migration must + never be the reason a database will not open. + + ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be + retried on every subsequent read of every table for the life of the process — the write path is + still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing + store call on the hot path of a grid that is trying to render. + """ + tenant = str(getattr(session, "tenant", "") or "") + if tenant in _IG_FORWARDED: + return + _IG_FORWARDED.add(tenant) + try: + import automation_engine as engine + engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") + + + +#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized +#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is +#: not — the shape this exists for is one already-paid provider payload per row. +JSON_LIST_MAX = 400 + + +def _thin_json(fields, merged, table_key=""): + """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. + + ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads + `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something + large rather than something empty. `_truncated` is what the viewer keys its fetch on. + + ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the + viewer needs no table key, no record id and no new props threaded down through three + components to find the document — the route that removed the value says where it went. One + writer of that address instead of a server rule and a client rule that must agree forever. + """ + json_keys = [str(f.get("key")) for f in (fields or []) + if str(f.get("type") or "") == "json"] + if not json_keys: + return merged + out = {} + for pid, cells in (merged or {}).items(): + row = cells + for key in json_keys: + raw = cells.get(key) + if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: + if row is cells: + row = dict(cells) + row[key] = json.dumps({ + "_truncated": True, "bytes": len(raw), + "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) + out[pid] = row + return out + + +@router.get("/tables/{table_key}/rows") +def table_rows(table_key: str, session: Session = Depends(require_session)): + """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, + identity}` — so the client's generic topic fetch consumes it with zero new parsing. + + ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell + from the OVERLAY stratum only (that is what makes a custom column render standalone) — a + user table's base values live in its DEFINITION rows, so they are layered UNDER the user's + overlay edits here: base first, overlay wins. Without this every base cell reads empty + (found by this route's own gate check, not by luck).""" + import aios_grid + + _ig_forward(session) + # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. + # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE + # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), + # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask + # about the SAME document in the SAME request. The lend is the fix W31 QA already built for + # `_defn_or_refuse`; this threads it through the three sites that never got it. + # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a + # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the + # value it replaced (contract C5). + _lent = _ut().lend(session.runtime) + g = ut_assembly(session, table_key, st=_lent) + # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge + # this line does not perform since the route was written (owner item 3, 2026-08-09: + # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). + # + # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` + # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table + # is overlay-typed by construction, so it looked up a key that could not be there and every + # such cell rendered blank. + # + # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds + # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed + # in the store exactly as they should. Only the READ-BACK dropped them, which is why the + # value survived the gesture, vanished on reload, and looked like "it reset itself" — and + # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. + # + # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition + # value must not shadow an edit the user has made on top of it. + # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this + # widens what a caller can SEE by exactly their own edits and nothing else. + _overlays = (g.get("ws") or {}).get("overlays") or {} + merged = {} + for _r in g["rows_src"]: + _pid = str(_r["pid"]) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _ov = _overlays.get(_pid) + if isinstance(_ov, dict): + _cells.update(_ov) + merged[_pid] = _cells + # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM GOES ON TOP, and the order is the contract rather + # than a preference. A shared column is one the whole permitted audience must read the SAME + # value in — that is the entire reason `shared_overlay` exists — so a stale per-user value + # left under the same key must not win. `routes_odoo_tables` layers the two in this exact + # order (`overlays.setdefault(pid, {}).update(cells)`) and the two doors must agree. + # ⛔ NEVER `setdefault` ON `merged`: these cells are already narrowed to this session's pids, + # but inventing a row id here would put a row in the payload that `rows_src` never admitted. + for _pid, _cells in (g.get("shared_cells") or {}).items(): + if str(_pid) in merged and isinstance(_cells, dict): + merged[str(_pid)].update(_cells) + # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE + # JSON DOCUMENTS DO NOT RIDE THE LIST. + # + # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% + # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB + # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The + # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole + # vendor response crossed the wire, was parsed by the browser and held in memory purely so a + # clipped first line could be drawn. + # + # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by + # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — + # the one place a person actually reads it, for the one row they opened. The cell that rides + # the list is a VALID small document saying what it stands for, so the preview renders + # honestly instead of showing half a truncated brace. + # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, + # so a tenant using `json` for a short config sees no change at all. + rows = aios_grid.rows_from_pool( + g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) + # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ + # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. + _stamp_ai_states(table_key, g["fields"], rows, st=_lent) + # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you + # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this + # database says so here, with its cause and the recommendation, rather than waiting to be + # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the + # honest answer for a table nothing limits — never an absent key, which a client cannot tell + # apart from an older server. + _report = _ut().limit_report(table_key, st=_lent) + # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the + # only thing that ever set this key; the write door never stopped working and every client + # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a + # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the + # customer scope — `grid_events.docs_for` — never a twin here. + from core import grid_events as _ge + return {"fields": g["fields"], "rows": rows, "today": g["today"], + "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, + admin=session.admin, st=session.runtime), + "pulled_at": time.strftime("%Y-%m-%d %H:%M"), + "identity": {"pid": "pid"}, + "scope": {"table": table_key, "rowCount": len(rows)}, + "limits": [_report] if _report else [], + "recordsMutable": _ut().records_mutable(table_key, st=_lent)} + + +#: The per-cell provenance a row carries on the wire, one key per enrichment column. +#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. +#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new +#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces +#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this +#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the +#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off +#: every field key, so no column can ever be called `_ai_*`. +AI_STATE_PREFIX = "_ai_" + + +def _stamp_ai_states(table_key, fields, rows, st=None): + """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. + + ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token + count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest + would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary + is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the + RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. + + ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no + enrichment column is untouched, and a freshly created column adds nothing until something + runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row + instead of against whatever was true when the value was written. + """ + cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] + if not cols: + return rows + import ai_enrich as _ae + for field in cols: + col = str(field.get("key") or "") + marks = _ut().ai_enrich_marks(table_key, col, st=st) + cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} + for row in (rows or []): + if not isinstance(row, dict): + continue + state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) + if state != "empty": + row[AI_STATE_PREFIX + col] = state + return rows + + +@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") +def table_cell(table_key: str, pid: str, fkey: str, + session: Session = Depends(require_session)): + """ONE cell, whole — the other half of `_thin_json`. + + ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor + for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON + viewer opens this for the one row a person is actually reading. + + ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's + view of the table), so this cannot become a side door onto a table the caller may not open — + which is the failure a "just fetch the raw cell" helper invites. + ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must + read back what they typed, not the definition value underneath it. + """ + g = ut_assembly(session, table_key) + field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) + if field is None: + raise err(404, "unknown field", f"{fkey!r} is not a column on this database") + row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) + if row is None: + raise err(404, "unknown record", f"no record {pid!r} in this database") + overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} + value = overlay.get(fkey, row.get(fkey)) + return {"table": table_key, "pid": str(pid), "field": str(fkey), + "value": "" if value is None else str(value)} + + +@router.post("/tables/{table_key}/rows", status_code=201) +def add_row(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). + + ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo + that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the + response rather than assume; the client re-anchors on what came back. + """ + _records_or_refuse(session, table_key) + ut = _ut() + values = (body or {}).get("values") or {} + if not isinstance(values, dict): + raise err(400, "bad_values", "values must be an object of {fieldKey: value}") + try: + rid = ut.add_row(table_key, values, session.uname, st=session.runtime, + rid=(body or {}).get("rid")) + except Exception: + raise err(503, "store_unavailable", "the row was not saved. The store refused") + if rid is None: + # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap + # sentence alone would misdirect — the reader would go and count rows. Ask the same + # validator the law used rather than re-deciding here (one rule, two voices). + pf = ut.profile_field(table_key, st=session.runtime) + if pf and pf["key"] in values: + _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) + if not ok: + raise err(400, "refused", + f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " + f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " + f"profile link (instagram.com/name)") + raise err(400, "refused", + f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") + _refresh_relations(session) + return {"rid": rid, "pid": int(rid)} + + +@router.post("/tables/{table_key}/rows/import", status_code=201) +def import_rows(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. + + Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a + spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, + nothing is matched or overwritten, and the dialog says so before the button is pressed. + + ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate + the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in + an import is not a stray to be tidied away — it means the client offered a target it should + not have, and silently dropping it would leave the user looking for a column of values that + never arrived. The refusal names the column. + + ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every + profile cell validates, because a half-imported file is the worst outcome available: the user + cannot tell which rows landed without reconciling the spreadsheet by hand. + """ + _records_or_refuse(session, table_key) + ut = _ut() + rows_in = (body or {}).get("rows") + if not isinstance(rows_in, list) or not rows_in: + raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") + if any(not isinstance(r, dict) for r in rows_in): + raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") + defn = _defn_or_refuse(session, table_key) + by_key = {f["key"]: f for f in (defn.get("fields") or [])} + asked = {k for r in rows_in for k in r} + unknown = sorted(k for k in asked if k not in by_key) + if unknown: + raise err(400, "unknown_field", + f"this database has no column {unknown[0]!r}") + computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) + if computed: + label = by_key[computed[0]].get("label") or computed[0] + raise err(400, "computed_field", + f"{label!r} is worked out from other columns, so it cannot be imported into") + # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. + # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a + # second client, or a future importer met no wall at all and the string landed verbatim in a + # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE + # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every + # rule on this route exists to prevent. The sentence names the row and the column, because + # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. + for index, row in enumerate(rows_in): + for key, value in row.items(): + why = ut.cell_type_refusal(by_key[key], value) + if why: + raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") + try: + made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "nothing was imported. The store refused") + if made is None: + raise err(400, "refused", + f"nothing was imported. {len(rows_in)} rows would take this database past " + f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") + _refresh_relations(session) + return {"imported": len(made), "pids": [int(r) for r in made]} + + +# --------------------------------------------------------------------------------------------- +# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) +# --------------------------------------------------------------------------------------------- +# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same +# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's +# definition to everyone without handing over the table. This supersedes wave 17's "fields are +# per-user" law for this path only; the connector scopes keep their own model. +# +# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the +# per-user workspace overlay, which is why the automation editor's "Automation column" picker +# could not see a column the user had just created — it reads the DEFINITION. Same defect shape +# as the rename (item 6a): two places to look, and the surfaces disagreed silently. + +def _field_or_refuse(session, table_key, fkey=""): + """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. + + ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and + `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they + write through the `user_tables` write doors), so no projected snapshot survives into a write. + ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 + draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" + defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + ut = _ut() + if not ut.is_user_table(table_key, st=session.runtime): + raise err(400, "not_a_user_table", + "only a user-created database has an editable schema. A connected source " + "owns its own columns") + # ⭐⭐ W36-T21 — A HIDDEN COLUMN IS NOT EDITABLE, AND THIS IS THE DOOR THAT HAD TO SAY SO. + # `EventCtx.hidden_keys` walls the events transport; the REST schema routes (rename, retype, + # delete a column) do not pass through it at all. Without this an account that could not SEE + # `unit_cost` could still DELETE it for the whole tenant — the loudest possible version of a + # wall that exists on one wire only. ⚠ Read the closure off the DECLARED fields, which is what + # `routes_admin` validates a stored `hiddenFields` list against. + if fkey and str(fkey) in _ut_hidden(session, table_key, defn.get("fields") or []): + raise err(403, "forbidden", + "this database has no column by that name that you may edit") + if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, + st=session.runtime): + field = next((f for f in (defn.get("fields") or []) + if f.get("key") == str(fkey)), None) + if isinstance((field or {}).get("automation"), dict) \ + and field["automation"].get("preset") is True: + # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not + # ut.may_edit_field` above is the wall). Said out loud because the sentence itself + # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a + # blanket "pre-set fields are locked" would now be the server explaining a refusal + # it did not make — `preset_editable` is the one predicate that answers this. + raise err(403, "preset_field_locked", + "this is a pre-set column, so its name and type are fixed; you may sort, " + "filter or hide it, edit any Rollup column, and add your own columns") + raise err(403, "forbidden", "that column can only be changed by the database's creator " + "or an admin") + if not fkey and not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can add a column") + return defn + + +@router.post("/tables/{table_key}/fields", status_code=201) +def add_field(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _field_or_refuse(session, table_key) + ut = _ut() + field = ut.add_field(table_key, body or {}, st=session.runtime) + if not field: + # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers + # None for every refusal, so this route said "check the name and type" to somebody whose + # name and type were fine and whose automation column named a flow that does not exist. + # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the + # one thing that was never wrong. Checked HERE, in the route's own words, because the + # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never + # re-implements it (a second copy of the rule is how two doors start disagreeing). + raise err(400, "refused", + _refusal_sentence(ut, session, body or {}, table_key=table_key)) + note = "" + if field.get("type") == "link": + synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) + field = synced.get("field") or field + # ⭐⭐ W41-T15 / C5 — WHY NO BACKLINK APPEARED, CARRIED BACK. A link into `customer_data` + # or `product_data` can never spawn one (that database has no definition document to hold + # a column), and until now the 201 was identical to the one a spawned backlink returns. + # Reported, never raised: the source link SAVED and resolves its own cell, so a 400 here + # would throw away work the user did over a convenience they did not ask for. + note = (synced.get("reason") or "") if synced.get("refused") else "" + # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route + # became reachable for one. It was gated on `link` alone, while `patch_field` and + # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first + # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). + # The user would have created the column, watched a 201 come back, and read a blank cell for + # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. + # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in + # the tenant, and adding a text column has nothing to fold. The condition is now "is this + # field relational", which is the question that was always meant. + if field.get("type") in ("link", "rollup"): + _refresh_relations(session) + out = {"field": field} + if note: + out["reciprocalNote"] = note + return _with_dropped(ut, out, body) + + +def _with_dropped(ut, out, body): + """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). + + ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag + cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and + `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the + defect. T51's contract is that an unknown config key is dropped **and named**, so the naming + rides the response beside the accepted field rather than inside the validator. + + ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to + ignore the key, which is how a report stops being read before it stops being true. + """ + dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) + if dropped: + out = dict(out) + out["dropped"] = dropped + return out + + +def _refusal_sentence(ut, session, body, table_key="", fkey=""): + """Why was this column refused? The specific reason when we can name one, the general list + otherwise — never a specific-sounding guess.""" + # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag + # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment + # column would otherwise be explained by the flow law -- "pick a flow, or make this an + # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user + # never touched. + if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": + bag = (body or {}).get("aiEnrich") + if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): + return ("an AI enrichment column needs a prompt. It is the only thing that can " + "produce a value here, so a column without one would stay empty forever") + # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is + # what the contract asks for and what makes it actionable — "at most one" sends the reader + # hunting through a 40-column schema for a flag they cannot see from the header. + if isinstance((body or {}).get("profile"), dict): + # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at + # once (an `int` profile column on a table that already has a profile column), and the + # TYPE is the one that is wrong about what the caller just sent — unconditionally, no + # matter what else is on the table. Answering "you already have one" to somebody whose + # real mistake was the column type sends them to fix the wrong thing, which is the + # misdirection D-46 closed one door over. + if str((body or {}).get("type") or "text").strip().lower() != "text": + return ("a profile column is a flag on an ordinary TEXT column. It validates what " + "is typed into it, which it can only do for text") + existing = ut.profile_field(table_key, st=session.runtime) if table_key else None + if existing and existing.get("key") != str(fkey): + return (f"this database already has a profile column: " + f"{existing.get('label') or existing.get('key')!r}. A database has at most " + f"one, so the automation knows which handle to enrich; edit that column, or " + f"take the flag off it first") + bag = (body or {}).get("automation") + if isinstance(bag, dict): + flow = str(bag.get("flowId") or "").strip() + if not flow: + return ("an automation column has to name the automation that fills it. Pick a " + "flow, or make this an ordinary column") + if not ut.flow_bound(bag, st=session.runtime): + return (f"this column names automation {flow!r}, which does not exist in this " + f"workspace. It may have been deleted; pick a flow that is still there") + kind = str((body or {}).get("type") or "").strip() + if kind and kind not in ut.UT_FIELD_TYPES: + return (f"{kind!r} is not a column type here (types: " + f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") + return (f"the column was refused. Check the name and type, or the table may be at its " + f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") + + +@router.patch("/tables/{table_key}/fields/{fkey}") +def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Edit one column's definition, and MIGRATE its values when options are renamed. + + ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: + [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed + Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and + every saved view that filtered on it. + """ + _field_or_refuse(session, table_key, fkey) + ut = _ut() + body = body or {} + migrated = None + renames = body.get("renames") + if renames: + try: + migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the rename did not land. Try again") + # The per-user workspace strata and any view filter naming the old value are the OTHER + # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an + # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a + # missing counterpart must not lose the half that DID land. + try: + import core.table_store as table_store + fn = getattr(table_store, "rename_choice_values", None) + if callable(fn): + fn(table_key, fkey, renames, st=session.runtime) + migrated = dict(migrated or {}, workspace=True) + except Exception: # noqa: BLE001 + migrated = dict(migrated or {}, workspace=False) + field = ut.patch_field(table_key, fkey, body, st=session.runtime) + if not field: + # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as + # adding one, so it must get the same sentence naming the column that already holds the + # flag — a patch that answered "check the name and type" would send the reader to the + # one thing that was never wrong (the D-46 lesson, one door over). + raise err(400, "refused", + _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) + synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) + field = synced.get("field") or field + _refresh_relations(session) + out = {"field": field} + # ⭐⭐ W41-T15 / C5 — the same report `add_field` carries, on the door that RETARGETS a link. + # This one matters more: pointing an existing link at `customer_data` deletes the backlink the + # old target held (the stale-inverse sweep) and cannot make a new one, so the column silently + # disappears from a database the user is not looking at. + if synced.get("refused"): + out["reciprocalNote"] = synced.get("reason") or "" + if migrated is not None: + out["migrated"] = migrated + return _with_dropped(ut, out, body) + + +def _fire_on_change(table_key, pid, changed, session): + """Run any `on_change` enrichment column whose prompt names a cell that just moved. + + ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. + `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a + 20,000-row import into 20,000 whole-document reads on the single process this product runs + (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and + takes the definition, so this reads once and asks about every column. + + ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; + an enrichment that could not run is a missing value, not a lost edit, and the run's own report + carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a + fan-out here would put a vendor call on the critical path of every keystroke-commit. + """ + import ai_enrich as _ae + + try: + defn = _ut().get(table_key, st=session.runtime) or {} + wanted = _ae.on_change_fields(defn, changed.keys()) + for field in wanted: + # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still + # somebody's edit spending somebody's tokens, so it is booked against the person who + # typed rather than left unattributed. + _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], + user=session.uname) + except Exception: # noqa: BLE001 + pass + + +@router.post("/tables/{table_key}/fields/{fkey}/enrich") +def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. + + ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides + (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the + tenant by opening a grid. + + `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the + value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic + plan, where the policy and the never-overwrite-a-human law both apply. The two are one + function with one flag, not two runners. + + ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by + reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a + ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a + populated `skipped` is a correct, informative answer, and the client must render it rather + than treat it as success. + """ + _field_or_refuse(session, table_key, fkey) + import ai_enrich as _ae + + rows = (body or {}).get("rows") + if rows is not None and not isinstance(rows, list): + raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " + "rows this column's own settings choose") + # The caller's permitted pool, the same one the row doors use. A named row outside it is + # dropped rather than refused: a stale client naming a record that has been deleted or moved + # out of scope should not fail a run over the rows it can legitimately fill. + if rows is not None: + allowed = {str(p) for p in scoped_pids(session, table_key)[0]} + rows = [str(r) for r in rows if str(r) in allowed] + # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" + # sends `always`. Anything else falls back to the column's own saved policy rather than to a + # default, so a typo cannot quietly widen what a run touches. + report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, + policy=str((body or {}).get("scope") or "") or None, + # A named row set through THIS door is a person asking. + manual=rows is not None, + # ⭐ W35-T41 / C7 — the usage ledger's attribution. + user=session.uname) + if report.get("problem"): + # A run that could not start at all is not a 200: nothing was attempted, nothing was + # spent, and the reason is actionable (no provider configured, or the wrong column). + raise err(400, "enrich_refused", str(report["problem"])) + return report + + +@router.delete("/tables/{table_key}/fields/{fkey}") +def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): + _field_or_refuse(session, table_key, fkey) + if not _ut().delete_field(table_key, fkey, st=session.runtime): + raise err(400, "refused", + "that column could not be removed. A database must keep at least one") + _refresh_relations(session) + return {"deleted": fkey} + + +@router.delete("/tables/{table_key}/rows/{rid}") +def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): + # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. + # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* + # A single DELETE was FIVE whole-document deep copies before the commit even began — three in + # the guard (`get` + `may_open` + `records_mutable`) and two more inside + # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads + # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** + # (D-185), so the guard alone was seconds of copying to answer questions about one row. + # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` + # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document + # under the store lock, so a lent snapshot can never be the thing written back. + # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually + # consistent delete is indistinguishable from one that did not work. This makes the guard + # cheap; it does not make the commit optimistic. + lent = _ut().lend(session.runtime) + _records_or_refuse(session, table_key, st=lent) + # ⭐⭐ W36-T21 — THE ROW SCOPE, ON THE DELETE DOOR. `patch_row` has asked this since it was + # written (`pid not in g["pids"]` -> 403) and this door never did, because until now every + # account that could open a `ut_*` database could see every row of it. The moment an + # administrator can row-scope one, "may not SEE row 5" and "may DELETE row 5" become two + # different answers unless this is here — and delete is the one that cannot be undone. + # ⚠ Guarded on `row_scope_applies` so an unscoped account pays nothing: for them the pid set + # is every row and the question has one answer. + import core.perm_scope as _ps + if _ps.row_scope_applies(session.user, table_key): + _pids, _f, _d = scoped_pids(session, table_key) + if not str(rid).isdigit() or int(rid) not in _pids: + raise err(403, "out_of_scope", "that row is not in this database") + try: + ok = _ut().delete_row(table_key, rid, st=lent) + except Exception: + raise err(503, "store_unavailable", "the delete did not land. Try again") + if not ok: + raise err(400, "refused", "rows can only be deleted from user-created databases") + _refresh_relations(session) + return {"ok": True} + + +@router.patch("/tables/{table_key}/rows/{pid}") +def patch_row(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Cell edits — the products PATCH on the user-table ctx. Routed through + `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; + the accepted values are read BACK from the bucket, never echoed from the request.""" + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + _records_or_refuse(session, table_key) + g = ut_assembly(session, table_key, consume_corrections=False) + if pid not in g["pids"]: + raise err(403, "out_of_scope", "that row is not in this database") + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, + # ⭐⭐ W36-T21 — the assembly's OWN closure, not an empty set. `g["fields"]` is already + # stripped, but `grid_events` asks `ctx.hidden_keys` by name: a PATCH naming a hidden + # column would otherwise be accepted on a payload that never showed it. + hidden_keys=g.get("hidden") or frozenset(), + st=session.runtime, # R6b (D-16) + scope_key=table_key, table=_ops(session, table_key)) + try: + grid_events.handle_one( + {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable. Your change was not saved") + # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. + # + # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH + # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. + # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's + # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began + # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` + # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring + # claiming otherwise is what makes the next reader look for a merge bug that is not here. + # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY + # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` + # asks whether THIS write landed, and this write goes to the definition. + stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) + .get(str(pid)) or {}) + accepted = {k: stored.get(k) for k in updates if k in stored} + + def _took(k): + """Did the cell TAKE this write? Normally that is "stored == asked". + + ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` + and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell + the client to roll back a write that landed. But it cannot simply be exempted either — + a junk handle leaves the OLD value sitting in `stored`, which would then read as + accepted. So the question asked is the exact one: **is what is stored the canonical form + of what was asked?** Anything else is a genuine refusal. + """ + if k not in accepted: + return False + want = str(updates[k]) + if stored.get(k) == want: + return True + pf = _ut().profile_field(table_key, st=session.runtime) + if pf and pf["key"] == k: + handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) + return bool(ok) and stored.get(k) == handle + return False + + refused = sorted(k for k in updates if not _took(k)) + # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write + # this cell?" is not recoverable from the value afterwards, so the only place to record it is + # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic + # run overwrite it, whatever the column's `overwrite` policy says. + # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused + # write `human` would freeze a cell against the agent on the strength of an edit that never + # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. + took = {k: v for k, v in accepted.items() if k not in refused} + if took: + try: + _ut().note_human_edit(table_key, took, pid, st=session.runtime) + except Exception: # noqa: BLE001 + # Provenance is metadata about a write that has already succeeded. Failing the + # request here would tell the user their edit was lost when it was not. + pass + _fire_on_change(table_key, pid, took, session) + out = {"pid": pid, "updates": accepted} + if refused: + out["refused"] = refused + # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a + # profile blank cleared. Without this the grid keeps painting a stale follower count under + # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other + # half: the client may keep only what the server actually took, and must be TOLD what else + # moved. Derived by diffing this row against what was asked for, so it cannot drift from + # whatever the clear rule decides to touch. + also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} + cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) + if cleared: + out["cleared"] = cleared + # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, + # amendment A5; SESSION B built and gated the function and correctly declared it INERT until + # this line existed, citing [[flag-shipped-without-its-writer]]). + # + # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead + # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the + # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we + # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try + # it again, the account is back". Without this call that person has no way back at all, and + # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. + # + # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this + # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a + # human touched the handle cell. + # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- + # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must + # be readable in the log rather than degrade into "the re-arm quietly stopped working". + _pf = _ut().profile_field(table_key, st=session.runtime) + if _pf and _pf["key"] in updates: + try: + import automation_engine as _engine + _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") + _refresh_relations(session) + return out + + +# ── CONTRACT C1 (W36-T20): THE MIRROR READER ────────────────────────────────────────────────── +# ⭐⭐ R6. `core.perm_scope.scoped_table` is the ONE door to any database's rows and it answers for +# a MATERIALISED `ut_*` table entirely on its own — deliberately, so a cold process (E's sandbox +# subprocess, a worker, a gate) that never imported a route still gets the right answer. A +# READ-THROUGH grid is the one arm it cannot serve alone: those ten databases store no rows in the +# tenant document at all, and their rows live in the DuckDB mirror behind `routes_odoo_tables`, +# two layers above `core`. +# +# ⛔ ONE FETCH, NOT A SECOND ONE. This hands C1 the SAME `_read_through_rows` that `scoped_pool` +# and `scoped_pids` already share, so the rows a script sees through C1 are byte-for-byte the rows +# the grid sees — by construction, not by a second query that agrees today (W31-T20's argument, one +# caller further out). +# +# ⚠ AND `TooBigToMaterialise` BECOMES A REPORTED REFUSAL, NEVER A SHORT ANSWER. Standing rule 1's +# second sentence, and the shape is `_PID_SCOPE_LIMIT`'s so nothing downstream needs a second +# vocabulary for it. +def _c1_mirror_rows(table_key, field_keys, st): + """C1's mirror arm: `perm_scope.register_mirror`'s reader over `_read_through_rows`.""" + import core.perm_scope as perm_scope + + try: + return _read_through_rows(table_key, field_keys, rt=st) + except _too_big() as e: + raise perm_scope.Unresolvable( + cause=str(e), + recommendation=_PID_SCOPE_LIMIT["recommendation"], + subject="rows", effect="unresolved") from e + except RuntimeError as e: + raise perm_scope.Unresolvable( + subject="rows", effect="unreadable", cause=str(e), + recommendation="the connector mirror is not ready on this process; retry once the " + "store has finished opening") from e + + +def _register_mirror(): + """Declare the mirror reader to C1. Called at import; returns True once it is registered.""" + import core.perm_scope as perm_scope + return perm_scope.register_mirror(_c1_mirror_rows) + + +_C1_MIRROR = _register_mirror() diff --git a/api/routes_web_agent.py b/api/routes_web_agent.py index 12364aa7911fe7e0042054e3a7f07e050fae7700..57453ded998810e7046aa5d14acae4d739fd235b 100644 --- a/api/routes_web_agent.py +++ b/api/routes_web_agent.py @@ -1,84 +1,84 @@ -"""routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51). - -The owner's words are the whole reason this file exists: *"we already laid the foundation of this -but never test anything."* The capability is otherwise reachable only from inside an automation -run, which means the first person to discover it is broken is a customer at 3am. - - GET /api/v1/web-agent/capability any session — can this deployment run a web step, and - if not, the SENTENCE saying why - POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what - came back, or the sentence - -⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract -C5); these are an operator surface over the same function, so a green test here and a red run -there cannot disagree about anything except the input. - -⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`, -so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody. - -⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's -network, never from this server, so this is not a door into our own infrastructure. It is still -admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems -the tenant is authorised to use, at their instruction") is not something an ordinary member -should be able to commit the tenant to. - -⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and -`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting -either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which -is the same class of stale claim as a green gate on a router nobody wired: three finished routers -once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. -""" -from fastapi import APIRouter, Body, Depends - -import web_agent -from deps import Session, require_session, err -from routes_admin import admin_gate - -router = APIRouter(prefix="/api/v1") - -MAX_URL = 2000 -MAX_SELECTOR = 400 - - -@router.get("/web-agent/capability") -def web_agent_capability(session: Session = Depends(require_session)): - """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`. - - Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say - if not") and withholds the deployment detail (namespace, which token key, the image) that - only an operator has any use for. Nothing here is ever a credential. - """ - cap = web_agent.capability() - return {"ready": bool(cap["ready"]), "reason": cap["reason"], - "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]} - - -@router.post("/web-agent/test") -def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)): - """Run ONE real `web_read` and report exactly what the seam returned. - - The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`, - or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something - went wrong" would hide the one thing it exists to show. - """ - url = str(body.get("url") or "").strip() - selector = str(body.get("selector") or "").strip() - if len(url) > MAX_URL or len(selector) > MAX_SELECTOR: - raise err(400, "too_long", "the URL or selector is longer than this door accepts") - - step = {"kind": "web_read", "id": "test", "url": url, "selector": selector, - "attr": (body.get("attr") or "text"), "all": bool(body.get("all")), - "timeoutMs": int(body.get("timeoutMs") or 20000)} - if body.get("waitFor"): - step["waitFor"] = str(body["waitFor"]) - - notes = [] - result, error = web_agent.run_step( - step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}", - "log": notes.append}) - if error: - # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the - # web step did not succeed. A 500 here would make an ordinary "the selector matched - # nothing" look like a server fault. - return {"ok": False, "error": error, "log": notes[-6:]} - return {"ok": True, "result": result, "log": notes[-6:]} +"""routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51). + +The owner's words are the whole reason this file exists: *"we already laid the foundation of this +but never test anything."* The capability is otherwise reachable only from inside an automation +run, which means the first person to discover it is broken is a customer at 3am. + + GET /api/v1/web-agent/capability any session — can this deployment run a web step, and + if not, the SENTENCE saying why + POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what + came back, or the sentence + +⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract +C5); these are an operator surface over the same function, so a green test here and a red run +there cannot disagree about anything except the input. + +⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`, +so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody. + +⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's +network, never from this server, so this is not a door into our own infrastructure. It is still +admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems +the tenant is authorised to use, at their instruction") is not something an ordinary member +should be able to commit the tenant to. + +⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and +`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting +either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which +is the same class of stale claim as a green gate on a router nobody wired: three finished routers +once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. +""" +from fastapi import APIRouter, Body, Depends + +import web_agent +from deps import Session, require_session, err +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +MAX_URL = 2000 +MAX_SELECTOR = 400 + + +@router.get("/web-agent/capability") +def web_agent_capability(session: Session = Depends(require_session)): + """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`. + + Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say + if not") and withholds the deployment detail (namespace, which token key, the image) that + only an operator has any use for. Nothing here is ever a credential. + """ + cap = web_agent.capability() + return {"ready": bool(cap["ready"]), "reason": cap["reason"], + "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]} + + +@router.post("/web-agent/test") +def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)): + """Run ONE real `web_read` and report exactly what the seam returned. + + The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`, + or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something + went wrong" would hide the one thing it exists to show. + """ + url = str(body.get("url") or "").strip() + selector = str(body.get("selector") or "").strip() + if len(url) > MAX_URL or len(selector) > MAX_SELECTOR: + raise err(400, "too_long", "the URL or selector is longer than this door accepts") + + step = {"kind": "web_read", "id": "test", "url": url, "selector": selector, + "attr": (body.get("attr") or "text"), "all": bool(body.get("all")), + "timeoutMs": int(body.get("timeoutMs") or 20000)} + if body.get("waitFor"): + step["waitFor"] = str(body["waitFor"]) + + notes = [] + result, error = web_agent.run_step( + step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}", + "log": notes.append}) + if error: + # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the + # web step did not succeed. A 500 here would make an ordinary "the selector matched + # nothing" look like a server fault. + return {"ok": False, "error": error, "log": notes[-6:]} + return {"ok": True, "result": result, "log": notes[-6:]} diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 78c940ada4d1f25f50cdd30e80bf5c6345ef567a..67e25acfdd80d20d49cfd3de417f63d7b0437a6b 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -643,7 +643,18 @@ def clean_measure_field(raw, offered): "custom": True, "derived": True, "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, + # W41-T21 (owner instruction 16): every numeric measure column arrives with a column + # summary, `pct` included. A pct measure used to fall through to `agg: None`, so a + # percentage column could never total at the bottom or per group. + # It defaults to AVERAGE rather than sum ON PURPOSE: twenty rows of 20% would "total" + # 400%, a number nobody can use. Currency and int keep `sum`. `average` is a member + # of FIELD_AGGS, which is the vocabulary the saved-field passthrough at the bottom + # of `fields_from_workspace` filters against, so it is not nulled on the way out. + # Do not tidy the pct arm back to `sum`. The `else None` arm stays fail closed for + # any future member of MEASURE_FIELD_TYPES that is not summable; today the `mtype` + # fallback above makes it unreachable. + "agg": ("average" if mtype == "pct" + else "sum" if mtype in ("currency", "int") else None), "note": str(raw.get("note") or "")[:2000], "measure": {"key": str(spec.get("key"))[:80], "window": window}, } @@ -735,7 +746,14 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ "custom": True, "derived": True, "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, + # W41-T21: the SAVED half of the same default, and it must stay identical to + # `clean_measure_field`. A measure column already sitting in somebody's + # workspace is rebuilt HERE, not there, so stamping only the create door would + # leave every existing pct column with no summary and ship the feature half + # dead. `pct` averages (a summed percentage is meaningless), currency and int + # sum. Do not tidy the pct arm back to `sum`. + "agg": ("average" if mtype == "pct" + else "sum" if mtype in ("currency", "int") else None), "note": str(field.get("note") or "")[:2000], "measure": {"key": mkey[:80], "window": window}, **_field_extras(field, mtype), @@ -2016,6 +2034,30 @@ def clean_folders(raw): icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark if icon: row["icon"] = icon + # W41-T12 / T33 / OWNER INSTRUCTION 12 -- `parent` SURVIVES THIS FUNCTION NOW. + # THIS ONE LINE IS WHY NESTING WAS DEAD. Every other half shipped and worked: + # `ViewSidebar.tsx:316` takes a `parent`, `CustomerGrid.tsx:7432` puts it on the wire, + # `table_store.validate_folder_tree` refuses cycles and chains over `MAX_FOLDER_DEPTH` + # with a worded receipt, and `grid_events` already catches `FolderTreeError` into + # `_folder_refuse`. This row build STRIPPED the key on the way in, so the validator + # could never fire and a folder made inside a folder was gone on refresh -- a feature + # dead behind four correct halves and every gate green. `grid_events.py` stated the + # forward contract itself: "the day `clean_folders` learns `parent`, the refusal is + # already wired and worded". This is that day. + # + # ABSENT, NEVER `None`, and never invented. An absent `parent` IS the root, which is + # what every stored row has meant until now -- and `validate_folder_tree` promises that + # a payload carrying no `parent` anywhere comes back byte-identical with zero repairs, + # so writing a null key would break that promise for no gain. + # SELF-PARENTING IS DROPPED HERE as well as refused downstream: the validator raises + # `cycle` on it, and a row that cannot be legal should not reach the validator wearing + # a repair. A parent naming a folder that is GONE is deliberately NOT dropped here -- + # deleting a folder creates exactly that, and `validate_folder_tree` repairs it to the + # root WITH a receipt the user can read. Dropping it silently here would destroy the + # receipt, and the user would never learn their folder had moved. + parent = str(f.get("parent") or "").strip()[:80] + if parent and parent != fid: + row["parent"] = parent items.append(row) items.sort(key=lambda x: x["order"]) for i, f in enumerate(items): diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index 2fe15d56a52d0b6e34cbae7dbe31d3a828841722..3aa93dda7386619c20904d40cb46d265fcc76f49 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -329,6 +329,14 @@ "default": true, "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." }, + { + "key": "product_id", + "label": "Odoo ID", + "type": "int", + "source": "odoo", + "default": false, + "description": "The Odoo product.product id, the key every Odoo document joins on. Carried on the row rather than derived, because a product's pid is a CRC32 of its SKU and the id cannot be recovered from it." + }, { "key": "product", "label": "Product", @@ -791,6 +799,140 @@ "default": false, "shared": true, "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace." + }, + { + "key": "sub_category", + "label": "Sub-Category", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Fine grained product class from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "main_category", + "label": "Main Category", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Top level product class from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "color", + "label": "Color", + "type": "multiselect", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Colors this SKU is offered in, from the Daily Mastersheet. Multi-select, because the sheet stores several colors in one comma separated cell and each one becomes its own value. Owned by the Administration account and shared with everyone in the workspace." + }, + { + "key": "shape_style", + "label": "Shape/Style", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Shape or styling of the piece, from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "material", + "label": "Material", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "What the piece is made of, from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "finish", + "label": "Finish", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Surface finish of the piece, from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "occassion", + "label": "Occassion", + "type": "multiselect", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Occasions this SKU is bought for, from the Daily Mastersheet. Multi-select, because the sheet stores several occasions in one comma separated cell and each one becomes its own value. The label keeps the mastersheet's own spelling. Owned by the Administration account and shared with everyone in the workspace." + }, + { + "key": "collection", + "label": "Collection", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "The merchandising collection this SKU belongs to, from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "size", + "label": "Size", + "type": "select", + "source": "overlay", + "default": false, + "preset": false, + "shared": true, + "createdBy": "admin", + "permissions": { + "edit": "collaborative" + }, + "description": "Size band from the Daily Mastersheet. Owned by the Administration account and shared with everyone in the workspace. Blank means the mastersheet carries no value for this SKU." + }, + { + "key": "upc", + "label": "UPC", + "type": "text", + "source": "odoo", + "default": false, + "description": "The SKU's barcode, read live from Odoo as product.product.barcode. Blank when Odoo carries no barcode for this product. The Daily Mastersheet has a UPC column of its own and it is deliberately not used: three quarters of it is a placeholder rather than a number." } ] } diff --git a/platform/core/field_permissions.py b/platform/core/field_permissions.py index 0b885522a1cefe2fed9e080b7014ec14e5ede0bd..f31053f92521918b783a06843f9162b9abd2423c 100644 --- a/platform/core/field_permissions.py +++ b/platform/core/field_permissions.py @@ -6,6 +6,8 @@ that could never be true. This module is the single normalization point for the personal, collaborative, or specific users. """ +import time + import core.shared_overlay as shared_overlay import core.shares as shares @@ -14,6 +16,28 @@ FIELD_EDIT_MODES = ("personal", "collaborative", "users") MAX_FIELD_USERS = 50 _ALIASES = {"everyone": "collaborative", "creator": "personal", "admins": "personal"} +#: ⭐⭐ W41-T01 / RULING R5 / CONTRACT C1 — THE THREE BADGES' VOCABULARIES, DECLARED ONCE. +#: R5: *"A field is classified by THREE badges, and every right derives from them: Origin +#: (`Pre-set` = the platform made it / blank = a person made it) x Audience (`Private` / +#: `Shared by ` / `Everyone`) x Values (`Shared values` = one value everyone sees / blank = +#: yours alone)."* A consumer greps ONE place for the spellings; nothing derives a badge +#: independently. +FIELD_ORIGINS = ("preset", "user") +FIELD_AUDIENCES = ("private", "users", "everyone") +FIELD_VALUE_STRATA = ("shared", "personal") + +#: `permissions_from_grants`' edit vocabulary -> C1's audience vocabulary. The two answer the SAME +#: question in different words (who can reach this column), so this map is the whole of the +#: translation and there is no second derivation to drift from it +#: ([[one-evaluator-per-question]]). ⛔ It is a MAP rather than a branch on purpose: a mode this +#: dict does not name falls to `private`, which is the fail-closed direction. +_AUDIENCE_FROM_EDIT = {"collaborative": "everyone", "users": "users", "personal": "private"} + + +def _uname(value): + """One spelling of a username, everywhere in this module.""" + return str(value or "").strip().lower() + def clean_permissions(raw, fallback="personal", known_users=None): """Return a small, canonical field permission bag or ``None`` for malformed input.""" @@ -115,6 +139,269 @@ def permissions_from_grants(entries): return clean_permissions({"edit": "users", "users": sorted(users)}) +# --------------------------------------------------------------- W41-T01 / R5 / C1: the badges +def _governed(field): + """Does the per-field grant WALL apply to this column? + + ⛔ ONE EVALUATOR, AND IT IS `perm_scope`'s. `granted_field_keys` is the cheap half of + `field_grant_hidden`; asking it here is what makes the BADGE and the WALL incapable of + disagreeing, which is this ticket's own acceptance. Re-spelling `f.get('granted') is True` + locally would be a second reader of the mark, and a second reader is how the two come apart. + + ⛔ THE IMPORT IS LAZY, MATCHING `field_grant_hidden`'s OWN `import core.shares` AT ITS CALL + SITE. `perm_scope` pulls `core.perms` -> `context`/`registry`/`users`; this module is imported + lazily from `grid_events` precisely to stay cheap, and a top-level import here would widen its + graph for every importer to buy nothing. + """ + try: + import core.perm_scope as perm_scope + except Exception: # noqa: BLE001 + # Unresolvable means "not governed": the badge then reports the column's OTHER, true + # properties instead of asserting a permission state it cannot verify. The wall itself + # does not degrade with it — it lives in `perm_scope` and is unreachable from here. + return False + return bool(perm_scope.granted_field_keys([field] if isinstance(field, dict) else [])) + + +def field_origin(field): + """R5's FIRST badge: `"preset"` (the platform made this column) or `"user"` (a person did). + + ⛔ READ OFF MARKS THAT ALREADY EXIST, AND NO NEW ONE IS INVENTED. There are exactly two kinds + of positive evidence that a PERSON made a column, and both are already written by the create + doors: `custom: True` (`grid_events.field_upsert`) and `createdBy` (stamped at create and + preserved from `prior` on every later write). A contract-declared column carries neither — + `aios_grid_fields.json` names only `key/label/type/source/default/description/shared/...` — so + Supplier and every canonical Odoo column answer `"preset"` without a mark being added to the + contract file. + + ⚠ `automation.preset` IS CHECKED FIRST AND IT WINS. It is the platform's own positive mark + (`automation_engine.ut_ensure(lock_fields=True)`), so a seeded column that somehow also carries + an author stays `"preset"`. ⛔ AND THE `type != 'rollup'` LEG OF THE CLIENT'S + `isSchemaLocked` IS DELIBERATELY NOT REPEATED HERE: that leg answers *"may this be edited"* + (`user_tables.preset_editable` returns True for exactly `rollup`), which is a RIGHT derived + from the badge, not the badge. A preset rollup was still made by the platform. + + ⚠ ABSENCE FALLS TO `"preset"`, AND THE POLARITY IS THE CONSERVATIVE ONE. Rights derive from + this badge, so calling a platform column user-made is the direction that would offer somebody + a delete they must not have; calling a user column platform-made is cosmetic and visible. + ⛔ KNOWN GAP, OUT OF THIS TICKET'S FENCE: on a `ut_*` database `user_tables._clean_field` is a + strict allowlist that rebuilds `out` from scratch and keeps neither `custom` nor `createdBy` + (it does keep `automation`), so a user-created column there has no author mark left to read and + answers `"preset"`. That is a defect in the allowlist, not in this derivation, and it does not + reach `customer_data` or `product_data`, whose definitions keep both marks. + """ + if not isinstance(field, dict): + return "preset" + automation = field.get("automation") + if isinstance(automation, dict) and automation.get("preset") is True: + return "preset" + if field.get("custom") is True or _uname(field.get("createdBy")): + return "user" + return "preset" + + +def _shared_value_keys(table_key, st=None): + """The columns of `table_key` whose VALUES are one tenant-wide stratum.""" + key = str(table_key or "").strip() + if not key: + return set() + try: + return {str(k) for k in shared_overlay.fields(key, st=st)} + except Exception: # noqa: BLE001 + return set() + + +def _empty_record(): + return {"owner": None, "entries": []} + + +def _grant_record(field_key, table_key=None, grant_topic=None, st=None): + """ONE column's grant record, read straight from the registry.""" + topic = str(grant_topic or table_key or "").strip() + key = str(field_key or "").strip() + if not topic or not key: + return _empty_record() + try: + return shares.grants("field", shares.field_oid(topic, key), st=st) + except Exception: # noqa: BLE001 + return _empty_record() + + +def grant_records(grant_topic, st=None): + """`{field_key: {'owner', 'entries'}}` for one topic, on **ONE** read of `object_shares`. + + ⭐⭐ THE BATCH EXISTS FOR A MEASURED REASON, NOT FOR TIDINESS. `shares.grants` opens the + registry per call and `store.get` hands back `json.loads(json.dumps(...))` — a fresh deep copy + every time. Badging a customer grid is 40-60 columns, so the per-field door is 40-60 + serialise/deserialise round trips added to the hot read path. That is the exact shape + `grid_assembly`'s own W38-T20 note spent a ticket removing (*"three consumers want this dict on + one request, and letting each take its own copy is the shape D-214 spent a whole ticket + removing one document over"*). + + ⛔ THE ENTRIES GO THROUGH `shares._clean_entries`, WHICH IS NOT OPTIONAL AND IS NOT TIDYING. + That normaliser DROPS an entry whose role is not in `shares.ROLES`, and `permissions_from_grants` + checks only the user. Reading the bucket raw would let a junk-role entry make this badge say + `users` while `shares.role_for` — which reads through `grants()`, i.e. through the normaliser — + walls that same person out. A badge that disagrees with the wall is precisely what R5's + *"every right derives from them"* forbids. Same normaliser, therefore same answer. + + ⚠ THE TOPIC IS THE GRANT TOPIC, NOT THE BUCKET. `migrate_legacy_fields` already carries this + warning in full: `shared_key` names the store bucket (`customer_table_workspace__shared`) and + `grant_topic` names the registry namespace (`customer_data`). Handing this one the bucket finds + no record for any column and every badge silently reads as never-shared. + """ + topic = str(grant_topic or "").strip() + if not topic: + return {} + try: + bucket = (shares._st(st).get(shares.SHARES_KEY) or {}).get("field") or {} + except Exception: # noqa: BLE001 + return {} + if not isinstance(bucket, dict): + return {} + out = {} + for oid, rec in bucket.items(): + table, key = shares.split_field_oid(oid) + # `split_field_oid` FAILS CLOSED on junk (`(None, None)`), so an unparseable id is skipped + # rather than being attributed to whichever database is being read. + if table != topic or not key or not isinstance(rec, dict): + continue + out[key] = {"owner": (_uname(rec.get("owner")) or None), + "entries": shares._clean_entries(rec.get("entries"))} + return out + + +def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, + values_shared=None, st=None): + """⭐⭐ W41-T01 / R5 / CONTRACT C1 — **THE ONE PRODUCER OF A FIELD'S THREE BADGES.** + + Returns `{origin, audience, sharedBy, values, owner}` and nothing else derives a badge + independently. The five member spellings are C1's and they are the wire's: the payload key is + `class`, its members are `origin`, `audience`, `sharedBy`, `values`, `owner`. + + `viewer` is a USERNAME STRING (`session.uname`), never a user dict — a dict compares unequal to + every stored name, which would make `sharedBy` silently never match and `owner` never equal the + reader. + + **origin** — `field_origin`, above. + + **audience** — `private` | `users` | `everyone`, and it MIRRORS THE WALL rather than paraphrasing + it, because a badge that says something the wall does not enforce is worse than no badge: + + * a column carrying `perm_scope.FIELD_GRANT_MARK` is GOVERNED, so the registry answers, + through the existing `permissions_from_grants` (extended, never duplicated): a `*` grant is + `everyone`, named grants are `users`, no grant at all is `private`; + * an UNGOVERNED column is not walled by `field_grant_hidden` at all, so whoever holds the + database holds the column. It is therefore `everyone` when the column is tenant-wide — a + contract-declared one (Supplier, and every canonical Odoo column) or one living in the + shared stratum — and `private` when it is not, because a per-user definition exists only in + its creator's own stratum and no other account ever receives it. + + ⭐ SUPPLIER IS THE CASE THE THIRD BADGE EXISTS FOR, AND IT IS WHY THE CONTRACT LEG IS NOT + OPTIONAL. It answers `everyone` while having NO `object_shares` row at all — its audience comes + from the contract that declared it, not from the registry, which is exactly why `ShareDialog` + prints "Not shared with anyone yet." about a column the whole workspace can already read. + + ⛔ THE TRAP, AND IT IS THIS TICKET'S OWN: `shared_overlay`'s `"shared": True` IS WRITTEN AND + NEVER READ (D-414). It is not consulted anywhere here. `FIELD_GRANT_MARK` is the only flag the + wall reads, so it is the only flag this function reads, and tenant-wide-ness is answered by + MEMBERSHIP (`values_shared`) rather than by that stale boolean. + + ⚠ A ROUTE COLUMN MINTED BEFORE THE MARK EXISTED THEREFORE READS `everyone`, AND THAT IS THE + TRUTH RATHER THAN A MISS. Such a column carries `shared: True` alone, is not `granted`, and + genuinely leaks to the whole tenant; the badge reports the leak instead of papering over it. + Closing it is W41-T05's work, and this badge is how it becomes visible. + + **sharedBy** — who shared it TO this viewer, or `None`. `shares.grants` carries `by` per entry + (D-474) and the owner is the fallback when a pre-D-474 record has none. `None` when the viewer + IS the owner, when nobody shared it, or when the viewer shared it to themselves. + ⚠ IT IS EMITTED UNDER AN `everyone` AUDIENCE TOO, not only under `users`. R5 renders the three + as alternative states of one badge, so a consumer that only wants a name for `Shared by ` + reads it under `users` and ignores it otherwise — but the fact ("alice let you in") is true in + both, and withholding it would make the producer decide a rendering question that belongs to + the consumer. + + **values** — `shared` when this column's VALUES are one stratum everyone sees (membership of + `shared_overlay.fields()` for an overlay topic, or a module's `SHARED_KEYS()` for a + contract-declared one), else `personal`. This is R5's new information and it is INDEPENDENT of + audience: a route column is `private` with `shared` values, Supplier is `everyone` with `shared` + values, and a column you made is `private` with `personal` ones. + ⚠ `values_shared` IS LENT BY THE CALLER because the key set is a MODULE-level fact + (`product_data.SHARED_KEYS()`), and `core` never imports up. Absent, it is derived from + `shared_overlay.fields(table_key)`, which is right for an overlay topic and blind to a + contract-declared one. + + **owner** — the registry's owner, falling back to the definition's `createdBy` when the + registry has no record. ⚠ THE FALLBACK IS NOT BELT-AND-BRACES: the create door writes the + definition FIRST and claims the grant SECOND, on purpose, so the window where a column exists + unclaimed is real. `field_grant_hidden` already relies on exactly this fallback to stop a + creator being walled out of the column they just made. + """ + field = field if isinstance(field, dict) else {} + key = str(field.get("key") or "").strip() + me = _uname(viewer) + + origin = field_origin(field) + record = grants if isinstance(grants, dict) else _grant_record( + key, table_key=table_key, grant_topic=grant_topic, st=st) + owner = _uname(record.get("owner")) or None + entries = [e for e in (record.get("entries") or ()) if isinstance(e, dict)] + + if values_shared is None: + values_shared = _shared_value_keys(table_key, st=st) + values = "shared" if key and key in {str(k) for k in values_shared} else "personal" + + if _governed(field): + audience = _AUDIENCE_FROM_EDIT.get( + permissions_from_grants(entries).get("edit"), "private") + elif values == "shared" or origin == "preset": + # Ungoverned AND tenant-wide: the database's holders all hold this column. `origin` covers + # a contract-declared column whose VALUES are per-user (a canonical Odoo column); the + # `values` leg covers a shared-stratum column the contract never named. + audience = "everyone" + else: + audience = "private" + + shared_by = None + if me and me != owner: + mine = next((e for e in entries if _uname(e.get("user")) == me), None) + room = next((e for e in entries if _uname(e.get("user")) == shares.EVERYONE), None) + entry = mine if mine is not None else room + if entry is not None: + shared_by = _uname(entry.get("by")) or owner + if shared_by == me: + shared_by = None + + return {"origin": origin, "audience": audience, "sharedBy": shared_by, + "values": values, "owner": owner or _uname(field.get("createdBy")) or None} + + +def field_classes(fields, viewer, *, table_key=None, grant_topic=None, values_shared=None, + st=None): + """`{field_key: }` for a whole field list, on ONE registry read. See `grant_records`. + + ⚠ EVERY COLUMN GETS A BAG, INCLUDING ONE THE REGISTRY HAS NEVER HEARD OF: the missing record is + substituted explicitly rather than left to `field_class` to re-read, which is what keeps the + single read single. + """ + topic = str(grant_topic or table_key or "").strip() + records = grant_records(topic, st=st) + if values_shared is None: + values_shared = _shared_value_keys(table_key, st=st) + shared_keys = {str(k) for k in (values_shared or ())} + out = {} + for field in (fields or ()): + if not isinstance(field, dict): + continue + key = str(field.get("key") or "").strip() + if not key: + continue + out[key] = field_class(field, viewer, grants=records.get(key) or _empty_record(), + table_key=table_key, grant_topic=topic, + values_shared=shared_keys, st=st) + return out + + def _same_classification(left, right): """Do two canonical bags say the same thing about WHO? @@ -213,6 +500,149 @@ def reconcile_shared_permissions(shared_key, grant_topic, st=None): return rewritten +# ═════════ W41-T07 / OWNER INSTRUCTION 21 — A DELETED FIELD STAYS DELETED ═══════════════════ +# +# Owner, verbatim: *"Deleting a field from Hide fields does not stick; 'August Campaign' on Odoo +# customer has reappeared three times."* +# +# ⛔⛔ MEASURED IN-PROCESS BEFORE ANY OF THIS WAS WRITTEN, because the scout listed the cause as a +# HYPOTHESIS and building on an unverified one is how a wave ships a correct fix for the wrong +# defect. `promote_field` (and the promotion arm of `migrate_legacy_fields` below) POPS the +# definition out of its creator's stratum, so on a PROMOTED column `table_store.delete_field` is a +# **total no-op**: the creator's `fields` map no longer holds the key, the shared bucket is a +# different document its `shared=` callback cannot reach, and every other account's fork is +# untouched. Observed: `anna['fields'] changed? False`, `shared_overlay STILL holds it? True`. +# The column the owner deleted three times was never deleted once. +# +# ⛔ AND `shared_overlay.drop_field` ALONE DOES NOT FIX IT, also measured: drop the shared +# definition, run ONE read, and `migrate_legacy_fields` promotes it straight back out of a fork +# sitting in another account's workspace. That is the fifth leg, and it is why the delete has to +# be a SWEEP of every stratum rather than a call to any one door. +# +# ⚠ WHICH FORKS RE-PROMOTE, measured, because the guard is untestable without it: a fork carrying +# NO permissions bag re-promotes (`stored_permissions` falls back to `collaborative`), one saying +# `collaborative` re-promotes, and one saying `personal` does not. A negative control built on a +# `personal` fork proves nothing at all: it would stay deleted with every guard disabled. + +#: The tenant-wide member of a `_table_workspace` document — `core.table_store.SHARED_KEY`. +#: ⚠ SPELLED, NOT IMPORTED, and deliberately: `table_store` imports nothing from this module and +#: an import the other way would close a cycle in `core`. `migrate_legacy_fields` has skipped this +#: member by the same literal since it was written, so the constant names what was already here. +WORKSPACE_SHARED_MEMBER = "__shared__" + +#: Where a delete records itself: `__shared__.deletedFields = {field_key: }`. +#: +#: ⭐⭐ THE RESIDENCY IS THE WHOLE COST ARGUMENT. `migrate_legacy_fields` ALREADY reads the +#: workspace document (`document = st.get(key)`) and a delete ALREADY writes it (the fork sweep), +#: so the tombstone is read for free on every read path and written in the SAME transaction as the +#: sweep it belongs to. One update, one flush, one failure mode — `table_store._update`'s own +#: argument for taking a `shared=` callback instead of making a second write. +#: +#: ⛔ IT IS A SIBLING OF `__shared__.fields`, NOT AN ENTRY IN IT, and that is what keeps it out of +#: everyone's way. `table_store._shared_fields` reads `__shared__['fields']`; `shared_views` reads +#: `__shared__['views']`; `assert_annotation_only` judges only the entries of `__shared__['fields']` +#: — so nothing that reads this member can see this book, and the C3 annotation law has nothing to +#: bite on. ⛔ AND IT IS NOT IN THE SHARED-OVERLAY BUCKET, which was the other candidate: that +#: document carries every shared CELL in the tenant and `store.get_projection` drops keys from +#: top-level VALUES rather than top-level keys, so reading it for a tombstone would buy a whole +#: deep copy of the cells on every read — the D-214 shape, paid on the hot path, for a dozen bytes. +DELETED_FIELDS_MEMBER = "deletedFields" + +#: How long a tombstone suppresses the IMPLICIT legacy promotion. See `delete_field_everywhere` +#: for why a finite window is the anti-leak, not a weakness. +TOMBSTONE_TTL_S = 900 + +#: The book is compacted on every write, so this is a ceiling on a document nobody prunes. +MAX_TOMBSTONES = 200 + + +def _live_tombstones(document, now=None): + """`{field_key: deleted_at}` for the deletes still inside their window, from a document the + caller ALREADY HOLDS. Costs no read, and never writes. + + ⚠ EXPIRY IS READ-SIDE FILTERING, NOT A SWEEP. Retiring an expired entry would be a write on a + read path, on a tenant-wide document, triggered by opening a page — the exact shape + `reconcile_shared_permissions` had to argue its way past. An expired entry is INERT here and is + compacted by `_stamp_tombstone` on the next delete, which is a write somebody asked for. + """ + member = (document or {}).get(WORKSPACE_SHARED_MEMBER) + book = member.get(DELETED_FIELDS_MEMBER) if isinstance(member, dict) else None + if not isinstance(book, dict): + return {} + now = int(time.time()) if now is None else int(now) + out = {} + for field_key, at in book.items(): + # `bool` IS an `int` in Python, so it is excluded by name: `{'k': True}` would otherwise + # read as "deleted at epoch 1" and be permanently expired, i.e. a silently absent guard. + if isinstance(at, bool) or not isinstance(at, (int, float)): + continue + if now - int(at) < TOMBSTONE_TTL_S: + out[str(field_key)] = int(at) + return out + + +def _stamp_tombstone(shared_member, field_key, now): + """Record one delete in the `__shared__` member being written. Compacts as it goes. + + ⚠ A SEPARATE MODULE-LEVEL FUNCTION RATHER THAN A CLOSURE, so the fifth-leg guard can be + DISABLED for a negative control without editing the delete. A guard nothing can turn off is a + guard nobody can prove is doing the work. + """ + book = shared_member.get(DELETED_FIELDS_MEMBER) + book = book if isinstance(book, dict) else {} + fresh = {str(k): int(v) for k, v in book.items() + if not isinstance(v, bool) and isinstance(v, (int, float)) + and now - int(v) < TOMBSTONE_TTL_S} + fresh[str(field_key)] = int(now) + if len(fresh) > MAX_TOMBSTONES: + for stale in sorted(fresh, key=lambda k: fresh[k])[:len(fresh) - MAX_TOMBSTONES]: + fresh.pop(stale, None) + shared_member[DELETED_FIELDS_MEMBER] = fresh + return fresh + + +def _retire_tombstone(document, field_key): + """Forget one delete, because somebody has explicitly said the column should exist again.""" + member = (document or {}).get(WORKSPACE_SHARED_MEMBER) + book = member.get(DELETED_FIELDS_MEMBER) if isinstance(member, dict) else None + if isinstance(book, dict): + book.pop(str(field_key), None) + return document + + +def _purge_forks(document, field_key): + """Drop one column's DEFINITION and VALUES from EVERY account's stratum. Returns the owners hit. + + ⛔⛔ EVERY ACCOUNT, WHICH IS THE ONE THING A PER-USER DELETE CANNOT DO AND THE REASON THIS + EXISTS. `table_store.delete_field(username, key)` reaches exactly one member of this document. + A fork left in any other member is re-promoted into the shared stratum by + `migrate_legacy_fields` on the very next read, and the column is back — measured, not reasoned. + ⚠ Deleting a COLUMN is already a tenant-wide act (`table_store.delete_field`'s own note makes + that argument about the tenant-wide summary), so reaching across accounts here is the same + act, not a wider one. The RIGHT to do it was answered before this function was reached. + + ⚠ MODULE-LEVEL AND NOT A CLOSURE, for `_stamp_tombstone`'s reason: the negative control has to + be able to switch this off and watch the column come back. + """ + key = str(field_key or "") + owners = [] + for owner, workspace in list((document or {}).items()): + if owner == WORKSPACE_SHARED_MEMBER or not isinstance(workspace, dict): + continue + touched = False + fields = workspace.get("fields") + if isinstance(fields, dict) and fields.pop(key, None) is not None: + touched = True + overlays = workspace.get("overlays") + if isinstance(overlays, dict): + for values in overlays.values(): + if isinstance(values, dict) and values.pop(key, None) is not None: + touched = True + if touched: + owners.append(str(owner)) + return owners + + def _field_is_migratable(field): return (isinstance(field, dict) and field.get("custom") is True and field.get("source") == "overlay") @@ -229,7 +659,8 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None grant_key = str(grant_topic or key).strip() shared_key = str(shared_key or key).strip() if not key or st is None: - return {"promoted": 0, "normalized": 0, "reclassified": 0} + return {"promoted": 0, "normalized": 0, "reclassified": 0, + "suppressed": 0, "purged": 0} # ⭐⭐ W40-T03 — RECONCILE THE SHARED STRATUM'S CLASSIFICATION FIRST, IN ITS OWN SWALLOWING # GUARD. This is the ONE seam that reaches every shared-field surface: all three readers # (`routes_customers.shared_fields`, `routes_tables._ut_shared_fields`, @@ -254,15 +685,21 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None try: document = st.get(key) or {} except Exception: - return {"promoted": 0, "normalized": 0, "reclassified": reclassified} + return {"promoted": 0, "normalized": 0, "reclassified": reclassified, + "suppressed": 0, "purged": 0} if not isinstance(document, dict): - return {"promoted": 0, "normalized": 0, "reclassified": reclassified} + return {"promoted": 0, "normalized": 0, "reclassified": reclassified, + "suppressed": 0, "purged": 0} promoted = 0 normalized = 0 + suppressed = 0 remove = {} + # ⭐⭐ W41-T07 — THE FIFTH LEG, READ FOR FREE OFF THE DOCUMENT ALREADY IN HAND. A key deleted + # inside the window is not promotable, no matter which account still holds a fork of it. + tombstoned = _live_tombstones(document) for owner, workspace in document.items(): - if owner == "__shared__" or not isinstance(workspace, dict): + if owner == WORKSPACE_SHARED_MEMBER or not isinstance(workspace, dict): continue fields = workspace.get("fields") or {} if not isinstance(fields, dict): @@ -270,6 +707,15 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None for field_key, original in list(fields.items()): if not _field_is_migratable(original): continue + if str(field_key) in tombstoned: + # ⛔ SKIP, NEVER SWEEP. The delete already took every fork it could see; anything + # here now either arrived after it or survived a partial failure, and destroying + # somebody's column to tidy up is the one mistake with no undo — this store keeps + # no history ([[a-cleanup-deletes-what-it-did-not-create]]). Suppressing the + # PROMOTION is enough: the fork stays private and visible to its holder, and the + # tenant-wide resurrection the owner reported three times does not happen. + suppressed += 1 + continue permissions = stored_permissions(original, fallback="collaborative", known_users=known_users) if permissions["edit"] == "personal": @@ -327,10 +773,42 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None values.pop(field_key, None) return data st.update(key, _remove, flush="sync") + + # ⭐⭐ W41-T07 — AND A SHARED DEFINITION THAT CAME BACK INSIDE THE WINDOW IS DROPPED AGAIN. + # + # ⛔ THIS IS THE LEG THAT MAKES A PARTIAL DELETE HEAL FORWARD, and it is why + # `delete_field_everywhere` sweeps the forks BEFORE it drops the shared definition. If the + # sweep and the tombstone land and the `shared_overlay.drop_field` after them does not, the + # definition would otherwise survive and be re-merged onto every grid forever. With this leg + # the next read finishes the delete instead. It also closes the concurrency window: a read + # that snapshotted the forks before the sweep can still write the promotion after it, and + # nothing else would ever take that resurrection away. + # + # ⚠ IT COSTS A READ OF THE SHARED BUCKET, AND ONLY WHEN A TOMBSTONE IS LIVE. Steady state is + # an empty book and this whole block is skipped, so the hot read path is unchanged; the cost + # is bounded to the `TOMBSTONE_TTL_S` window after somebody deletes a column. + purged = 0 + if tombstoned: + try: + live = shared_overlay.fields(shared_key, st=st) + except Exception: # noqa: BLE001 + live = {} + for dead in tombstoned: + if dead not in live: + continue + try: + if shared_overlay.drop_field(shared_key, dead, st=st): + purged += 1 + except Exception: # noqa: BLE001 + continue + # ⛔ `promoted` AND `normalized` KEEP THEIR NAMES AND THEIR MEANINGS. # `aios-web/api/verify_field_permissions.py` asserts `result["promoted"] == 1`; renaming or # folding either into the new count would turn a green gate red for a change it never made. - return {"promoted": promoted, "normalized": normalized, "reclassified": reclassified} + # ⚠ `suppressed` and `purged` are ADDED beside them for the same reason: a caller that reads + # the old two keeps reading exactly what it read before. + return {"promoted": promoted, "normalized": normalized, "reclassified": reclassified, + "suppressed": suppressed, "purged": purged} def promote_field(workspace_key, shared_key, grant_topic, owner, field, st=None): @@ -391,6 +869,13 @@ def promote_field(workspace_key, shared_key, grant_topic, owner, field, st=None) for values in (ws.get("overlays") or {}).values(): if isinstance(values, dict): values.pop(key, None) + # ⭐⭐ W41-T07 — AN EXPLICIT SHARE RETIRES THE TOMBSTONE, AND THIS LINE IS THE WHOLE + # ANTI-LEAK ARGUMENT FOR HAVING ONE. `delete_field_everywhere` suppresses the IMPLICIT + # legacy promotion of a deleted key; this function is the EXPLICIT one, reached from + # `routes_shares.put_share`, i.e. a person saying in as many words that this column is + # meant to exist and be shared. There is nothing left to be careful about after that, so + # the suppression ends immediately rather than waiting out `TOMBSTONE_TTL_S`. + _retire_tombstone(data, key) return data st.update(workspace_key, _remove, flush="sync") return shared @@ -424,3 +909,93 @@ def demote_field(workspace_key, shared_key, grant_topic, field, st=None): shared_overlay.drop_field(shared_key, key, st=st) shares.drop_objects([("field", shares.field_oid(grant_topic, key))], st=st) return personal + + +def delete_field_everywhere(workspace_key, shared_key, grant_topic, field_key, st=None): + """⭐⭐ W41-T07 / OWNER INSTRUCTION 21 — **DELETE ONE COLUMN FROM EVERY STRATUM THAT HOLDS IT.** + + Owner: *"Deleting a field from Hide fields does not stick; 'August Campaign' on Odoo customer + has reappeared three times."* It reappeared because no door had ever deleted it. Returns a + per-leg report; it decides no rights and refuses nobody. + + ⛔⛔ THE RIGHT IS ANSWERED BEFORE THIS IS REACHED, AND IT IS NOT ANSWERED AGAIN HERE. + `user_tables.may_delete_field` / `delete_field_refusal` are C2's one pair, and C2's last + sentence is *"No surface re-implements either test"* — a module that both decided and executed + would be the second implementation. This is the EXECUTION half, on purpose. + + **THE FIVE LEGS, and which line closes each:** + + 1. the deleter's own per-user stratum -> `_purge_forks` (their member of the document) + 2. `shared_overlay`'s bucket -> `shared_overlay.drop_field`, the RIGHT document + 3. the re-merge onto the next fetch -> leg 2; `_merge_shared_fields` and + `_ut_shared_fields` project what is in the bucket, + so a definition that is gone cannot be re-injected + 4. the grant record -> `shares.drop_objects` + 5. every OTHER account's fork -> `_purge_forks` (all members) + `_stamp_tombstone` + + ⛔ LEG 5 IS THE ONE THAT BITES, AND `drop_field` ALONE IS NOT ENOUGH — measured in-process + before this was written. `migrate_legacy_fields` runs on EVERY read and walks EVERY member of + the workspace document, so one fork left in one colleague's stratum re-promotes the column into + the shared bucket on the next page load. Deleting the shared definition without sweeping the + forks buys exactly one render. + + ⛔ THE ORDER IS SWEEP-THEN-DROP AND IT IS LOAD-BEARING, not stylistic. The sweep and the + tombstone land in ONE transaction on the workspace document; only then is the shared definition + dropped. So a failure of the second half leaves a live tombstone, and the next read's + `migrate_legacy_fields` finishes the delete instead of re-merging the survivor forever. The + other order (drop first, sweep second) fails the opposite way: the definition is gone, the + forks are not, and the very next read puts it back with nothing recording that anyone objected. + + ⚠ THE `__shared__.fields` COLUMN SUMMARY GOES TOO, for `table_store.delete_field`'s own stated + reason: *"an orphan `agg` under a deleted key is state nobody can see and nobody can clear, and + it would attach itself to the next column that happens to take the key back."* + """ + key = str(field_key or "").strip() + ws_key = str(workspace_key or "").strip() + bucket = str(shared_key or "").strip() + topic = str(grant_topic or bucket or "").strip() + report = {"key": key, "personal": 0, "owners": [], "shared": False, + "grants": False, "tombstoned": False} + if not key or st is None: + return report + now = int(time.time()) + + if ws_key: + hit = {"owners": []} + + def _purge(data): + data = data if isinstance(data, dict) else {} + hit["owners"] = _purge_forks(data, key) + member = data.setdefault(WORKSPACE_SHARED_MEMBER, {}) + if isinstance(member, dict): + annotations = member.get("fields") + if isinstance(annotations, dict): + annotations.pop(key, None) + _stamp_tombstone(member, key, now) + return data + + # ⛔ `flush='sync'`, matching every other STRUCTURAL write in this module. A delete the + # user has confirmed must not be the thing that is still coalescing when the process dies; + # `shared_overlay._write`'s own note draws the same line ("a lost column definition is a + # worse failure than a lost keystroke") and a lost DELETION is worse again, because the + # column comes back and the person deletes it a fourth time. + st.update(ws_key, _purge, flush="sync") + report["owners"] = list(hit["owners"]) + report["personal"] = len(hit["owners"]) + report["tombstoned"] = True + + if bucket: + report["shared"] = bool(shared_overlay.drop_field(bucket, key, st=st)) + + if topic: + # ⚠ GUARDED, AND THE TOLERANCE IS `routes_tables.delete_shared_field`'s ALREADY: the column + # and its values are gone by now, so a registry hiccup must not turn a completed delete + # into a 500 that invites the client to repeat it. A surviving grant record is inert + # (`shares.role_for` resolves against an object that no longer exists) and is overwritten + # by the next `set_grants` on the same id. + try: + shares.drop_objects([("field", shares.field_oid(topic, key))], st=st) + report["grants"] = True + except Exception: # noqa: BLE001 + report["grants"] = False + return report diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 0d1f1a4785637e58f0360a20cb050d44999f8725..1965535b253e17e9c6d9886bcde34107f3f8a4ed 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -626,6 +626,122 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True): uname, {'views': {}, 'fields': {}, 'overlays': {}}) +#: ⭐⭐ W41-T14 · OWNER INSTRUCTION 11 · CONTRACT C9 — THE GRANTED FOLDER'S OWN RECORD. +#: +#: ⛔ THE ICON WAS NEVER LOST IN TRANSIT; IT WAS NEVER SENT. `_granted_views` stamped +#: `v['sharedFolder'] = ` and nothing else, so `folders.ts::groupByFolder` RECONSTRUCTS +#: `{id: sharedFolderGroupId(owner, name), name}` for the receiver's rail — a folder assembled out +#: of two strings, with no icon on it — and `icons.tsx::FolderMark` then falls back to its default +#: shape and tone. That fallback is exactly what the owner is reporting. The record was always in +#: hand: `table_store.find_folder` returns `dict(hit)`, the whole stored row, icon included. +#: +#: ⚠ ADDITIVE, AND THAT IS THE CONTRACT DECISION. `sharedFolder` KEEPS its string type and its +#: meaning: `aios-web/web/verify_folders.py` asserts `typeof v.sharedFolder === "string"` and +#: `api/verify_api.py` reads the same key, so retyping it to a record would turn two gates red and +#: blank the receiver's grouping until the client half (W41-T35) lands. C8's rule one bullet above +#: C9 — *absent keys mean "not built yet", never "false"* — is what lets the two halves land in +#: either order, and [[two-lanes-one-contract-dead-feature]] is the scar it exists to prevent. +#: +#: ⛔ THE CLIENT MUST NOT USE THIS `id` AS THE GROUP KEY. `folders.ts::SHARED_FOLDER_PREFIX` says +#: it in its own words — *"NOTHING MAY EVER WRITE ONE… no `order`, no icon, no row in `folders`"* — +#: and every rail affordance gates on `isSyntheticFolderId`. Swapping the synthetic group id for +#: this real one would let the receiver emit `item_move` / `folder_rename` against a folder that +#: lives in somebody ELSE's store. `id` is here to identify the share, never to key the group. +#: +#: ⛔ `parent` IS DELIBERATELY NOT FORWARDED, which is why C9 spells it `parent?`. A parent id +#: names a folder in the OWNER's stratum that was NOT granted: the receiver cannot resolve it, +#: must not store it, and would gain only a pointer into a tree they cannot see. One folder is +#: shared, so one folder is delivered, flat. +#: +#: ⚠ BUILT MEMBER BY MEMBER, never `dict(folder)`. This row comes out of another account's +#: workspace; forwarding it whole would carry whatever else happens to be sitting on it across a +#: user boundary. The icon goes back out through `aios_grid.clean_folder_icon` — the ONE server +#: whitelist (12 shapes, 5 tones, and the legacy `grey` tone canonicalised) — so a receiver can +#: never be handed a shape no client knows how to draw. +#: ⭐⭐ W41-T20's DONE-WHEN CLAUSE 5, LANDED BY THE INTEGRATOR. Lane B proved BOTH ways that a +#: saved pivot round-trips to `None` without this, and correctly refused to write it: this file +#: is lane A's fence. The clause named a file its own ticket did not own, so it was +#: undischargeable by construction. [[two-lanes-one-contract-dead-feature]] +#: +#: ⛔ THE KEY IS WRITTEN UNCONDITIONALLY, like `important` above and unlike `cohortLock`. The +#: client CLEARS a pivot by omitting the key (`CustomerGrid.tsx:7195` +#: `const { secondaryView: _cleared, ...rest } = config`), so both spellings happen to clear +#: correctly today -- but this dict is REBUILT FROM THE ALLOWLIST on every autosave, and a +#: conditional key is one client change away from a pivot you can set and never clear. Lane A's +#: own note at :1331-1334 argues this for `important`; every word of it applies here. +#: +#: ⛔ FAIL-CLOSED ON `database` AND `viaField`, matching `types.ts::cleanSecondaryView` exactly: +#: a pivot missing either cannot be served by anything, so it stores as ABSENT (the ordinary view +#: of this table) rather than as a pivot that renders an empty grid reading as +#: "no related records" -- the distinction C10's `refusal` key exists to preserve. +#: +#: ⚠ THE NESTED `config` IS BOUNDED, NOT KEY-VALIDATED, AND THAT IS DELIBERATE. Its filters name +#: columns of the TARGET database, whose field list is not resolvable from this door -- `cfg`'s +#: `valid_keys` belong to the SOURCE. `routes_grid.py::pivot` already takes `filters` from the +#: request body as a raw list (`:1201`) and owns their semantics, so validating here would either +#: duplicate that or reject a legal tree. What this does own is SHAPE and SIZE, so nothing +#: unbounded reaches the store. +_SECONDARY_WINDOWS = ('all', 'last12m', 'ytd') # routes_grid.PIVOT_WINDOWS' key set + + +def _clean_secondary_view(raw): + """C6 -- the stored shape of a Relational pivot, or None. Mirrors `types.ts::cleanSecondaryView`.""" + if not isinstance(raw, dict): + return None + database = str(raw.get('database') or '').strip()[:80] + via_field = str(raw.get('viaField') or '').strip()[:120] + if not database or not via_field: + return None # fail-closed: see the note above + window = raw.get('window') + cfg = raw.get('config') if isinstance(raw.get('config'), dict) else {} + height = cfg.get('rowHeightMode') + def _keys(value, cap): + return [str(k)[:120] for k in list(value or [])[:cap] if isinstance(k, str)] \ + if isinstance(value, list) else [] + widths = {} + if isinstance(cfg.get('widths'), dict): + for k, v in list(cfg['widths'].items())[:400]: + try: + widths[str(k)[:120]] = int(v) + except (TypeError, ValueError): + continue + return { + 'database': database, + 'viaField': via_field, + 'window': window if window in _SECONDARY_WINDOWS else 'all', + 'config': { + 'filters': list(cfg.get('filters') or [])[:200] if isinstance(cfg.get('filters'), list) else [], + 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and', + 'sorts': list(cfg.get('sorts') or [])[:20] if isinstance(cfg.get('sorts'), list) else [], + 'groupBy': str(cfg['groupBy'])[:120] if isinstance(cfg.get('groupBy'), str) and cfg['groupBy'] else None, + 'colorBy': str(cfg['colorBy'])[:120] if isinstance(cfg.get('colorBy'), str) and cfg['colorBy'] else None, + 'rowHeightMode': height if height in ('medium', 'tall') else 'short', + 'order': _keys(cfg.get('order'), 400), + 'visible': _keys(cfg.get('visible'), 400), + 'widths': widths, + }, + } + + +def _shared_folder_record(fid, folder): + """C9's wire record for a folder granted to somebody else: `{id, name, order?, icon?}`. + + Absent `icon` means the owner chose none and the client draws its default mark, which is the + same thing absence has always meant on a stored folder row (`aios_grid.clean_folders`). + """ + import aios_grid as _agf + src = folder if isinstance(folder, dict) else {} + rec = {'id': str(fid or '')[:80], 'name': str(src.get('name') or '')[:80]} + try: + rec['order'] = int(src.get('order')) + except (TypeError, ValueError): + pass # the owner's rail position rides when it is one, never invented + icon = _agf.clean_folder_icon(src.get('icon')) + if icon: + rec['icon'] = icon + return rec + + def _granted_views(ctx, uname): """{view_id: STAMPED view} for every view GRANTED to `uname` by name (wave 21, item 9, C1). @@ -710,6 +826,14 @@ def _granted_views(ctx, uname): # renders in the same "Shared with me" group the view leg feeds. Stamps live on the # PROJECTION only and are never written back, exactly as above. v['sharedFolder'] = str(folder.get('name') or '')[:80] + # ⭐⭐ W41-T14 (C9) — AND NOW THE WHOLE RECORD BESIDE IT, which is the line that + # carries the icon. See `_shared_folder_record`: `sharedFolder` is untouched so the + # existing readers and their two gates keep working, and `sharedFolderRecord` is the + # key W41-T35 reads the shape and tone off. Same projection-only rule as every stamp + # above — `views_from_defs` copies a saved view with `dict(saved)` and `scope_view` + # with `dict(view)`, so an unknown key rides all the way to the wire untouched, and + # the `view_upsert` save path builds from a fixed whitelist so it can never be stored. + v['sharedFolderRecord'] = _shared_folder_record(fid, folder) out[str(vid)] = v return out @@ -852,14 +976,23 @@ def _known_usernames(): #: `code` is a stable machine token (the client may branch on it); `message` is the sentence a #: person reads. ⚠ The message must name what the CALLER can change — a refusal a user cannot act #: on is only marginally better than silence. -def _refuse(ctx, code, message, key=''): +#: +#: ⛔⛔ W41-T13 — `event` USED TO BE THE HARDCODED LITERAL `'field_upsert'`, and that was fine only +#: while this seam had exactly one caller. It now has two, so a folder refusal filed through the +#: old body would have recorded `{'event': 'field_upsert'}` on an `item_move` — a disclosure +#: channel built to end unfalsifiable theories, telling the next reader the wrong thing about +#: which door refused. KEYWORD-ONLY WITH THE OLD LITERAL AS THE DEFAULT, deliberately: every +#: existing `field_upsert` call site is byte-unchanged, and W41-T02's own refusals keep reporting +#: exactly what they reported before. The default is load-bearing, not decorative — it is what +#: makes this a widening rather than a rewrite. +def _refuse(ctx, code, message, key='', *, event='field_upsert'): """Record a NAMED refusal on the result and demand a repaint. Always returns True.""" try: out = getattr(ctx, 'out', None) if out is not None and hasattr(out, 'refusals'): if len(out.refusals) < 24: # one per event in the window; never unbounded out.refusals.append({ - 'event': 'field_upsert', + 'event': str(event or 'field_upsert')[:40], 'key': str(key or '')[:80], 'code': str(code)[:40], 'message': str(message)[:300], @@ -872,6 +1005,142 @@ def _refuse(ctx, code, message, key=''): return True +#: ⭐⭐ W41-T13 — THE CHANNEL THAT ACTUALLY REACHES A SCREEN, and the two are not the same one. +#: `EventResult.refusals` is assembled by `api/routes_grid.py` into `results[].refused` and a +#: top-level `refusals` array — and MEASURED 2026-08-24, **nothing under `aios-web/web/src` reads +#: either key**. `apiBridge.ts` (~1461) unpacks exactly `{results, doc, toast, rerender, derived}` +#: from a 200, so `toast` is the only member of that envelope a person ever sees. +#: +#: So a refusal that is only recorded is still a silent 200 to the user, which is the defect this +#: ticket names. Both are written: `refusals` is the machine record a caller branches on and the +#: next investigation reads, `toast` is the sentence. +#: +#: ⚠ `toast` IS ONE SLOT, read once by `routes_grid` after its whole per-event loop, and it is +#: SHARED with `add_to_list`'s confirmation. THE PRECEDENCE IMPLEMENTED HERE: a refusal always +#: claims the slot, so within one window the last refusal is what is shown. THE LIMIT, stated +#: rather than left to be discovered: an `add_to_list` that lands AFTER a refused folder event in +#: the same 24-event window still overwrites it, because that branch assigns the slot directly. +#: Widening `EventResult` to a message LIST is the real fix and it is not this ticket's file. +def _folder_refuse(ctx, kind, code, message, key=''): + """Refuse ONE folder event out loud: the machine record AND the rendered sentence.""" + try: + out = getattr(ctx, 'out', None) + if out is not None: + out.toast = str(message)[:300] + except Exception: # noqa: BLE001 + pass + # ⛔ RETURNS True, LIKE EVERY OTHER REFUSAL. `handle_one`'s return value means "repaint from + # server state"; a folder refusal that answered False would leave the browser's OPTIMISTIC + # placement on screen with nothing stored behind it — which is the springback the owner is + # reporting, kept for another two minutes instead of corrected at once. + return _refuse(ctx, code, message, key=key, event=kind) + + +#: ⭐ W41-T13 — WHICH `save_folders` REPAIRS ARE WORTH A SENTENCE. W41-T12's receipt lists every +#: correction it made; most mean *"something you sent was wrong and we fixed it"*, which a person +#: wants to know about because their folder tree now differs from what they dragged. +#: +#: ⛔ `parent_is_root_sentinel` IS DELIBERATELY ABSENT AND MUST STAY ABSENT. It means "you said +#: root the other legal way" — both spellings are correct, nothing was lost, and rendering it as a +#: warning would fire on ordinary use. A notice that cries wolf on the normal path trains the user +#: to ignore the ones that matter. +#: +#: ⚠ THE WHOLE NOTICE MUST FIT IN 300 CHARACTERS, which is where `EventResult.toast` is sliced. +#: All three phrases plus both joiners and the frame is the worst case, and the first draft came +#: to 328 — a sentence cut off mid-word, in the one channel a person actually reads. The phrases +#: below are short for that reason and not for style; the proof pins the bound so the next person +#: to add one finds out here rather than on a screen. +_FOLDER_REPAIR_COPY = { + 'parent_missing': 'named a folder that is gone, so it moved to the top level', + 'placement_missing_folder': 'named a folder that is gone, so it moved to the top level', + 'item_in_two_folders': 'was in two folders at once, so the first one was kept', +} +#: Everything else actionable collapses to one sentence: the row could not be read at all. +_FOLDER_REPAIR_UNREADABLE = frozenset({ + 'duplicate_folder_id', 'parent_not_an_id', 'row_not_a_folder', 'folder_without_id', + 'surface_not_a_list', 'placements_not_a_map', 'placement_without_item', + 'placement_not_an_id', +}) + + +def _folder_repair_notice(repairs): + """The sentence for a `save_folders` receipt, or None when there is nothing worth saying.""" + seen = [] + for row in (repairs or ()): + reason = str((row or {}).get('reason') or '') + if reason == 'parent_is_root_sentinel': + continue # see the constant above: the normal path, not a repair + phrase = _FOLDER_REPAIR_COPY.get(reason) + if phrase is None and reason in _FOLDER_REPAIR_UNREADABLE: + phrase = 'could not be read, so it was left out' + if phrase and phrase not in seen: + seen.append(phrase) + if not seen: + return None + return ('Your folders were saved with a correction. Something in them ' + + '; something else '.join(seen) + + '. Reload the page to see the result.') + + +def _commit_folder_write(tops): + """⭐⭐ W41-T13 / OWNER INSTRUCTION 10 — MAKE THE FOLDER WRITE THAT JUST HAPPENED DURABLE, NOW. + + ⛔ THE DEFECT, MEASURED WITH TWO REAL PROCESSES AGAINST A THROWAWAY STORE (2026-08-24): + `table_store.save_folders` commits through `TableStore._update`, which is hardcoded to + `flush='async'`. That applies the change to `Store._cache` and returns; the hub commit is + handed to a coalescing worker that sleeps `_FLUSH_DELAY` (2s) and then, if this key committed + inside the last `_FLUSH_MIN_GAP` (20s), sleeps the remainder of that too. **SIX accepted + folder writes in a row left the hub on its original commit** and a second process read an + empty `itemFolders` every single time. Read-your-writes held inside the process, which is + exactly why no gate in this repo could see it ([[a-guard-closed-on-one-backend-is-not-closed]]). + + The user-visible half is the owner's report: `viewEcho.ts::ECHO_RECENT_MS` holds the + optimistic placement for 120s and then yields to the host copy, so a write that never reached + the hub before the container was replaced springs back at about two minutes. + + ⛔ `st.flush(key)` IS THE WRONG PRIMITIVE HERE, THREE TIMES OVER: it waits out + `_FLUSH_MIN_GAP`, so it can block a drag for ~22 seconds; `harness.runtime.TenantRuntime` (the + handle the HTTP door actually passes) does not proxy it at all; and the gates' store doubles + do not all carry it. `update(key, identity, flush='sync')` is on EVERY handle — the module, + `Store`, `TenantRuntime`, `store_pg`, and both fakes — and `_update_locked`'s D-305 branch + makes it exactly the right one: with the key dirty and owned it takes the CACHE (never + re-downloading over the pending change), commits it through `_commit` with the parent + precondition and the replay journal intact, and clears the dirty mark so the worker does not + then spend a second commit on identical bytes. + + ⛔ BYPASSING `_FLUSH_MIN_GAP` IS THE POINT, NOT AN OVERSIGHT — do not "fix" this back to + protect the 256-commits/hr budget. That floor exists to coalesce autosave chatter (a column + resize, a filter tweak, a typed cell). A folder event is not chatter: `CustomerGrid.tsx` + emits `item_move` and `folder_reorder` ON DROP, one per completed gesture, so the ceiling + this adds is a handful of commits per session against a budget of 256 per hour. + + ⚠ IT IS THE HANDLE `save_folders` ITSELF USED, resolved through `_tops` and never `_store_of`. + Those two can be different objects (`_tops` can fall through to `cl_mod.TABLE_OPS`, built with + NO store handle, while `_store_of` returns the request's `TenantRuntime`), and committing the + wrong one would land nothing while looking exactly like success. `TableStore.st` and + `.table_key` are the public pair, so this addresses byte-for-byte what `_update` addressed. + + ⚠ A FAILED COMMIT IS NOT A REFUSED WRITE. The change is already accepted into the cache and + the key is still marked dirty, so the coalescing worker keeps retrying with backoff. Logging + and carrying on is therefore right; raising here would turn a saved change into a 500. + + ⚠ ON POSTGRES this is a redundant transaction: `store_pg.update` accepts `flush=` and ignores + it because every write is already committed when it returns. One extra RMW per user gesture, + correctness-neutral. Named for the record rather than special-cased, because a backend test + here would be a second definition of "which backend am I on". + """ + try: + st = getattr(tops, 'st', None) + key = getattr(tops, 'table_key', None) + if st is None or not key or not hasattr(st, 'update'): + return False + st.update(key, lambda doc: doc, flush='sync') + return True + except Exception as exc: # noqa: BLE001 + _tel.error('grid_events:folder_commit', exc) + return False + + def handle_events(events, ctx): """The event LOG entry point → one EventResult. The `[-24:]` window lives here. @@ -1146,6 +1415,8 @@ def handle_one(event, ctx): 'widths': widths, 'memberPids': [pid for pid in list(cfg.get('memberPids') or []) if isinstance(pid, int) and pid in allowed_pids], + # ⭐⭐ W41-T20 clause 5 / C6 -- see `_clean_secondary_view` above. + 'secondaryView': _clean_secondary_view(cfg.get('secondaryView')), } # Wave-6 item 10: a non-grid display mode rides the view config, validated # structurally (mode whitelist, refs must be this table's fields). @@ -1885,11 +2156,22 @@ def handle_one(event, ctx): # events touch ONE home — the table workspace's `folders` + `itemFolders` — so a cohort # can be filed without the cohort STORE learning about folders at all. import aios_grid as _agf + # ⭐⭐ W41-T13 — NOT ONE BARE `False` LEAVES THIS BRANCH ANY MORE. Every exit below was a + # `return False`, which `routes_grid` renders as `{"results":[{"rerender":false}]}` — byte + # for byte a success. The user dragged a view, the rail sprang back, and the app said + # nothing at all. `refused` carries the cause a caller branches on; `_folder_refuse` also + # claims the one slot the client renders (see its note). if not _store_of(ctx).available(): - return False + return _folder_refuse( + ctx, kind, 'store_unavailable', + 'Your folders could not be saved because the workspace store is not reachable ' + 'right now, so nothing was changed. Try again in a moment.') surface = str(event.get('surface') or '') if surface not in _agf.FOLDER_SURFACES: - return False + return _folder_refuse( + ctx, kind, 'surface_unknown', + 'That folder change did not name a sidebar this page has, so nothing was saved. ' + 'Reload the page and try again.', key=surface) # The ids this user may actually file: their own views, their own cohorts. Fail-closed — # an item_move naming somebody else's cohort is dropped by clean_item_folders anyway, # but refusing here means we never write it in the first place. @@ -1898,6 +2180,18 @@ def handle_one(event, ctx): return set((cur_ws.get('views') or {})) return set(_cohorts(ctx).all_for(uname)) + # ⚠ A CELL THE CLOSURE FILLS, because `_mutate` has ONE way to say no — `return None` — + # and eleven distinct reasons for saying it. Threading the cause out through the existing + # sentinel keeps every refusal at the line that decided it, rather than re-deriving the + # reason at the caller from state that has already moved on. + why = {} + + def _no(code, message, key=''): + """Refuse from inside `_mutate`, naming the cause. Returns None, the sentinel.""" + why.clear() + why.update({'code': code, 'message': message, 'key': str(key or '')}) + return None + def _mutate(cur_ws): folders = _agf.clean_folders(cur_ws.get('folders')) placed = dict(cur_ws.get('itemFolders') or {}) @@ -1908,24 +2202,76 @@ def handle_one(event, ctx): fid = str(event.get('folderId') or '').strip()[:80] if kind == 'folder_create': name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] - if not fid or not name or len(lst) >= _agf.MAX_FOLDERS: - return None + if len(lst) >= _agf.MAX_FOLDERS: + # ⚠ THE CAP IS SPLIT OUT FROM THE MALFORMED CASE, because it is the one a + # person can actually do something about and the only one they will ever hit. + return _no('folders_at_cap', + f'This sidebar already holds {_agf.MAX_FOLDERS} folders, which is ' + f'all it can show, so the new one was not created. Delete a folder ' + f'you no longer need, then try again.', key=fid) + if not fid or not name: + return _no('folder_needs_a_name', + 'A folder needs a name before it can be created, so nothing was ' + 'saved. Type a name and try again.', key=fid) if any(f['id'] == fid for f in lst): - return None + return _no('folder_already_exists', + 'A folder with that id is already on this sidebar, so nothing was ' + 'created. Reload the page to see the folders you have.', key=fid) row = {'id': fid, 'name': name, 'order': len(lst)} icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5) if icon: row['icon'] = icon + # OWNER INSTRUCTION 12 -- the client's `parent` reaches the store. Paired with + # `aios_grid.clean_folders`, which carries it through the sanitiser; NEITHER EDIT + # DOES ANYTHING ALONE. `clean_folders` runs over the whole list on every folder + # event, so a rename or a reorder cannot un-nest a folder as a side effect either. + # Bounded to 80 like every other id on this door, absent rather than null for the + # root, and self-parenting dropped. A parent naming a folder that has since been + # deleted is passed THROUGH on purpose: `validate_folder_tree` repairs it to the + # root and returns a receipt, which `_folder_repair_notice` already turns into the + # sentence the user reads. + parent = str(event.get('parent') or '').strip()[:80] + if parent and parent != fid: + row['parent'] = parent lst.append(row) elif kind == 'folder_rename': name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] hit = next((f for f in lst if f['id'] == fid), None) - if not hit or not name: - return None - hit['name'] = name + if not hit: + return _no('folder_gone', + 'That folder is not on this sidebar any more, so it could not be ' + 'renamed. Reload the page to see the folders you have.', key=fid) + # ⭐⭐ W41-T14 · OWNER INSTRUCTION 11 — AN ICON-ONLY EVENT IS LEGAL ON THIS DOOR. + # + # ⛔ THE SECOND HALF OF THE INSTRUCTION IS *"allow changing the icon from the + # '...' menu"* — a Change icon item, NOT the rename pane wave-9 built this branch + # for. A dialog that changes only the icon has no name to send, and until this + # line `{folderId, icon}` fell straight into the refusal below: the receiver of a + # gesture that never mentioned a name was told *"A folder needs a name"*, the icon + # was not stored, and both lanes were individually correct + # ([[two-lanes-one-contract-dead-feature]], with a wrong sentence on top). + # + # ⚠ THE DISCRIMINATION IS THE `name` KEY, NOT ITS EMPTINESS, and that is what + # keeps W41-T13's refusal intact: an event that CARRIES `name` and means blank is + # still a rename to nothing and is still refused, in T13's exact words. Today's + # client always sends one (`CustomerGrid.tsx::onFolderRename`), so every event + # that has ever been emitted takes the byte-identical path. An event with neither + # a usable name nor an `icon` key is malformed and is refused as it always was. + if not name and ('name' in event or 'icon' not in event): + return _no('folder_needs_a_name', + 'A folder needs a name, so the new one was not saved and the old ' + 'name was kept. Type a name and try again.', key=fid) + if name: + hit['name'] = name # wave-9 I15 (C5): the rename pane is also where the icon is changed. An event # that carries NO icon key leaves the existing one alone (a rename must not # silently strip a chosen icon); an explicit null clears it back to default. + # ⚠ W41-T14 MEASURED THE LIMIT RATHER THAN WIDENING IT: `clean_folder_icon` + # answers None for an explicit null AND for a junk shape, so this branch cannot + # tell *"clear my icon"* from *"that shape does not exist"* and drops the stored + # icon either way. Nothing junk is ever STORED, which is what this ticket owes; + # the conflation is booked rather than fixed here, because a new refusal would + # claim the single `toast` slot on the path W41-T35 is about to exercise. if 'icon' in event: icon = _agf.clean_folder_icon(event.get('icon')) if icon: @@ -1934,7 +2280,9 @@ def handle_one(event, ctx): hit.pop('icon', None) elif kind == 'folder_delete': if not any(f['id'] == fid for f in lst): - return None + return _no('folder_gone', + 'That folder is not on this sidebar any more, so there was nothing ' + 'to delete. Reload the page to see the folders you have.', key=fid) lst = [f for f in lst if f['id'] != fid] # CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that: # an item with no placement renders at the top level. @@ -1958,7 +2306,9 @@ def handle_one(event, ctx): # folder it merely failed to mention would be a data loss dressed as a sort. order = [str(i).strip()[:80] for i in (event.get('order') or [])] if not order: - return None + return _no('order_named_no_folders', + 'That rail arrangement named no folders, so the order you had was ' + 'kept. Drag the folder again.') known = {f['id']: f for f in lst} seen, ranked = set(), [] for fid_ in order: @@ -1967,17 +2317,34 @@ def handle_one(event, ctx): ranked.append(known[fid_]) ranked.extend(f for f in lst if f['id'] not in seen) if len(ranked) != len(lst): # cannot happen; refuse rather than truncate - return None + return _no('order_incomplete', + 'That rail arrangement could not be applied without losing a ' + 'folder, so the order you had was kept. Reload the page and try ' + 'again.') for i, f in enumerate(ranked): f['order'] = i lst = ranked elif kind == 'folder_duplicate': new_id = str(event.get('newId') or '').strip()[:80] src = next((f for f in lst if f['id'] == fid), None) - if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS: - return None + if len(lst) >= _agf.MAX_FOLDERS: + return _no('folders_at_cap', + f'This sidebar already holds {_agf.MAX_FOLDERS} folders, which is ' + f'all it can show, so the copy was not made. Delete a folder you ' + f'no longer need, then try again.', key=fid) + if not src: + return _no('folder_gone', + 'That folder is not on this sidebar any more, so there was nothing ' + 'to copy. Reload the page to see the folders you have.', key=fid) + if not new_id: + return _no('duplicate_without_id', + 'The copy was not given an id, so nothing was created. Try ' + 'duplicating the folder again.', key=fid) if any(f['id'] == new_id for f in lst): - return None + return _no('folder_already_exists', + 'A folder with that id is already on this sidebar, so the copy was ' + 'not made. Reload the page to see the folders you have.', + key=new_id) lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME], 'order': len(lst)}) # ⚠ The folder's CONTENTS are duplicated by the client emitting the ordinary @@ -1988,8 +2355,19 @@ def handle_one(event, ctx): else: # item_move item_id = str(event.get('itemId') or '').strip()[:120] target = event.get('folderId') - if not item_id or item_id not in _own_ids(surface, cur_ws): - return None + # ⭐⭐ W41-T13 / OWNER INSTRUCTION 10 — THE TWO CAUSES ARE SPLIT, and the split is + # the ticket's own argument: `not item_id` is a malformed event, `not in _own_ids` + # is a permission answer, and one `return None` for both told the user neither. + if not item_id: + return _no('move_without_item', + 'That move did not say which item to file, so nothing was saved. ' + 'Drag the item again.') + if item_id not in _own_ids(surface, cur_ws): + # ⚠ Worded for the person, not the wall: a view shared WITH someone is in + # their rail and is not theirs to file, which is the case they will meet. + return _no('item_not_yours', + 'You can file only the items you own, and this one belongs to ' + 'somebody else, so it stayed where it was.', key=item_id) cur = dict(placed.get(surface) or {}) if target is None: # ⚠ "UNFILE" — the client says nothing about where it went. Popping is right @@ -2015,7 +2393,10 @@ def handle_one(event, ctx): else: tid = str(target)[:80] if not any(f['id'] == tid for f in lst): - return None + return _no('folder_gone', + 'That folder is not on this sidebar any more, so the item ' + 'stayed where it was. Reload the page to see the folders you ' + 'have, then try again.', key=tid) cur[item_id] = tid placed[surface] = cur folders[surface] = lst @@ -2031,8 +2412,48 @@ def handle_one(event, ctx): # change inside a pure move. _res = _mutate(table_workspace(ctx, consume_corrections=False)) if _res is None: - return False # refused: nothing is written - _tops(ctx).save_folders(uname, _res[0], _res[1]) + # Refused: nothing is written — and now the caller is TOLD, with the cause `_mutate` + # recorded at the line that decided it. The fallback exists so a future `return None` + # added without a `_no(...)` degrades to a generic sentence rather than to silence. + return _folder_refuse( + ctx, kind, + why.get('code') or 'folder_change_refused', + why.get('message') or ('That folder change could not be saved, so nothing was ' + 'altered. Reload the page and try again.'), + key=why.get('key') or '') + # ⭐⭐ W41-T12's HANDOVER, TAKEN. `save_folders` VALIDATES before it writes and RAISES + # `FolderTreeError` on a cycle or an over-deep chain, having written nothing. Letting that + # escape would 500 the whole event window; catching it and dropping the result would put + # it straight back into the silent-200 path this ticket exists to close. `str(exc)` is + # already the sentence a person reads and already C7-clean, and `folder_id` is an + # attribute, so nothing is parsed out of the message. + # + # ⚠ UNREACHABLE FROM THIS DOOR TODAY, AND SAYING SO IS PART OF THE HANDOVER. + # `_mutate` returns `_agf.clean_folders(...)`, whose row is `{id, name, order}` plus an + # optional `icon` — it STRIPS `parent` entirely, so no nesting can reach the validator + # through here and neither raise can fire. This is the forward contract with T12: the day + # `clean_folders` learns `parent`, the refusal is already wired and worded. + try: + _receipt = _tops(ctx).save_folders(uname, _res[0], _res[1]) + except _tstore.FolderTreeError as _fte: + return _folder_refuse(ctx, kind, + str(getattr(_fte, 'reason', '') or 'folder_tree_illegal'), + str(_fte), key=str(getattr(_fte, 'folder_id', '') or '')) + # ⭐⭐ THE DURABILITY HALF — the owner's actual complaint. See `_commit_folder_write`: the + # write above lands in `Store._cache` and the hub commit is deferred by up to ~22s, so + # a second process (or the same one after a restart) cannot read it back. Forced here, + # for EVERY folder kind rather than `item_move` alone: they all write one key through one + # `save_folders`, they are all one discrete gesture, and a `folder_create` that evaporated + # would take the move filed into it with it. + _commit_folder_write(_tops(ctx)) + # W41-T12's receipt, partitioned. A repair means the stored tree differs from what was + # dragged, which the user has to be told or their sidebar silently disagrees with them. + _notice = _folder_repair_notice((_receipt or {}).get('repairs')) + if _notice: + try: + ctx.out.toast = _notice[:300] + except Exception: # noqa: BLE001 + pass # True: the sidebars render from server state (which folder holds what), not from local # optimism — the same contract the cohort panel's membership edits ride. return True @@ -2092,8 +2513,79 @@ def handle_one(event, ctx): import aios_grid as _ag3 if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)): return False + # ⭐⭐ W41-T02 / RULING R6a+R6b / CONTRACT C2 — THE RIGHT, ASKED OF THE ONE PAIR THAT OWNS + # IT. Until now this door had NO rights test at all: the prefix above was the whole of + # the wall, so any holder of a grid could delete a column somebody else made and shared + # with them, and an administrator could delete a pre-set the rename door had always + # refused them. C2's last sentence is *"No surface re-implements either test"*, so the + # answer comes from `user_tables.may_delete_field`, which derives it from C1's badges. + # + # ⚠ THE PREFIX TEST STAYS, AND IT IS NOT THE SAME WALL. It is STRUCTURAL — it says which + # STRATUM a key could even be deleted from, which is what stops the browser claiming a + # base contract column is deletable. The predicate is the RIGHTS wall over the columns + # that survive it. Replacing one with the other would drop a check in either direction. + # + # ⛔ NO `_refuse` HERE, ON PURPOSE. That helper stamps `'event': 'field_upsert'` on every + # record it writes, so reporting from this branch would file a refusal that lies about + # which event it was. The refusal is real and silent for exactly one more ticket: W41-T07 + # owns turning this door's `return False` into a truthy result with a message, and this + # line is what gives it a genuine refusal to carry. Until then the wall is proved from + # STORE STATE — the definition is still there after a refused delete — never from a + # refusal record this door cannot yet emit honestly. + # ⚠ THE DEFINITION IS RESOLVED IN TWO PLACES, AND THE SECOND IS NOT BELT-AND-BRACES. + # `field_by_key` is `ctx.fields`, bound ONCE per request, and it must be asked FIRST + # because it is the only one carrying the tenant-wide SHARED stratum (merged into the + # served contract by the assembly, never into this user's own bucket). But it is also + # STALE inside a batch: `handle_events` walks a window of events with one ctx, so a + # create and a delete arriving in the same burst would leave the delete looking at a + # field list assembled before the create landed — and the pair fails CLOSED, so the + # column would survive its own delete. `ws` is re-read per event a few lines up, so the + # per-user stratum it holds is current. Neither lookup DECIDES anything; both just hand + # the same one predicate the definition it is owed. + _defn = field_by_key.get(key) + if not isinstance(_defn, dict): + _defn = (ws.get('fields') or {}).get(key) + _share_key, _share_topic = _field_share_keys(ctx) + import core.user_tables as _ut2 + if not _ut2.may_delete_field( + _share_key, key, uname, bool(admin), + ctx.st or getattr(ctx.table, 'st', None), + field=_defn, grant_topic=_share_topic): + return False + # W41-T07 / OWNER INSTRUCTION 21 -- DELETE FROM EVERY STRATUM, NOT JUST THIS USER'S. + # Owner: *"Deleting a field from Hide fields does not stick; 'August Campaign' on Odoo + # customer has reappeared three times."* It reappeared because THIS door never deleted + # it: `table_store.delete_field` reaches exactly one member of the workspace document -- + # the deleter's own -- while the definition survives in `shared_overlay`'s bucket and in + # every other account's fork, and `migrate_legacy_fields` (which runs on EVERY read) + # promotes one straight back. Measured in-process by the ticket that built the executor: + # drop, one read, the column is back. + # + # W41-T07 BUILT `delete_field_everywhere` and wired it to `routes_tables.py:: + # delete_shared_field`, the generic `ut_*` door. THIS door -- the one serving + # `customer_data` and `product_data`, which is the grid the owner named -- was left on the + # single-stratum call, so the fix existed and the defect shipped anyway. A correct + # mechanism on the wrong door is why every gate stayed green. [[two-lanes-one-contract-dead-feature]] + # + # THE RIGHT WAS ALREADY DECIDED ABOVE and is not decided again here: C2's last sentence is + # *"No surface re-implements either test"*, and `may_delete_field` is that test. This is + # the EXECUTION half only, which is exactly what the executor's own docstring says it is. + # The keys are the pair `_field_share_keys` already resolved: `customer_data`'s per-user + # and tenant-wide strata are named from ONE key (`routes_customers::_shared_key` -- + # *"the store key this topic's per-user AND tenant-wide strata are both named from"*), so + # workspace and shared are the same argument here, and the grant topic is the second. if _store_of(ctx).available(): - _tops(ctx).delete_field(uname, key) + import core.field_permissions as _fp2 + _removed = _fp2.delete_field_everywhere( + _share_key, _share_key, _share_topic, key, + st=ctx.st or getattr(ctx.table, 'st', None)) + # The per-leg report is kept rather than reduced to a boolean: a delete that reached + # the shared bucket but no fork, or the reverse, is a different fact from a clean one. + try: + ctx.out.derived = dict(getattr(ctx.out, 'derived', None) or {}, + fieldDeleteRemoved=_removed) + except Exception: # noqa: BLE001 + pass else: _session_ready() ws['fields'].pop(key, None) diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index b8d8597073cd62ca6e89a6af06157d4f1c9194ea..32c6c5925e5e1f1cbd3655d08e12ea51785553e9 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -1,1619 +1,1926 @@ -"""core/perm_scope.py — the permission WALL for table modules (wave 15, C-PERM). - -ONE place answers the three questions a restricted account raises on a grid surface: - - may_access(user, module) may they open it at all? - visible_fields(fields, u, mod) which COLUMNS may they receive? - apply_row_scope(rows, u, mod, …) which ROWS may they receive? - -plus one that exists only because of how the pool is built: - - derive_pool_scope(user, module) which (team_id, agent) must the pool be BUILT with? - -Both hosts call these — `aios-web/api` (`grid_assembly`) and `app.py` (`_table_grid`) — because -a wall that exists on one runtime and not the other is not a wall. `core/perms.py` stays what it -is (module GRANTS + the legacy BU derivation); this module is the row/field/pushdown layer that -sits on top, and it is deliberately a separate file so the legacy readers can keep their -semantics untouched while this one fails closed. - -──────────────────────────────────────────────────────────────────────────────────────────── -THE RECORD - - user['perms'] = {'': {'access': bool, - 'filter': {'conj'?: 'and'|'or', 'nodes': [...]} | None, - 'hiddenFields': ['', ...]}} - user['perms_v'] = 1 stamped by the migration and by every write - -`perms_v` is the EXPLICIT-RESOLUTION marker, and it is here because of `permissioning.md` -Part II gap #5: "`allowed_modules() is None` is fail-open by default … make 'resolved: -unrestricted' an explicit value so absence/uncertainty DENIES." The same class already shipped -twice in this codebase (`modules: []` and `bus: []` both read as UNRESTRICTED — -`routes_admin.py`'s own docstring documents both). So: - - * `perms_v` ABSENT → the record is UN-MIGRATED, and the LEGACY wall applies unchanged - (`core.perms` module grants + the `bus`/`agent` query scope). That is not fail-open: it is - today's real wall, and it bounds the rollout window to "until the migration runs". - * `perms_v == 1` and the module has NO entry → **DENY**. Absence now means what it says. - * `role == 'admin'` bypasses all of it — which is also what keeps BREAK-GLASS alive. - `deps._user_for` hands back a hardcoded master dict on a store outage - (`{'username':'admin','role':'admin','bus':'all','modules':'all'}`) that will never carry a - perms block; without this clause an explicit-marker scheme locks the owner out of their own - product at exactly the moment the store is broken. -""" -import re - -import core.perms as perms - -#: `aios_grid._FORMULA_REF`'s pattern, restated rather than imported: this module is imported by -#: the API's request path and `aios_grid` pulls in the whole grid stack. Same regex, one line, -#: and `verify_api` asserts the two agree so it cannot drift into a different grammar. -_FORMULA_REF = re.compile(r"\{([^{}]*)\}") - -PERMS_VERSION = 1 - - -def _rec(user): - return user if isinstance(user, dict) else {} - - -def is_migrated(user): - """True once this record carries an explicit resolution. See the module docstring.""" - return int(_rec(user).get('perms_v') or 0) >= PERMS_VERSION - - -def entry(user, module): - """This user's declared permissions for `module`, or None if nothing is declared. - - None is AMBIGUOUS on purpose and every caller must resolve it against `is_migrated`: - on a migrated record it means DENY, on a legacy one it means "ask the old wall". - """ - p = _rec(user).get('perms') - if not isinstance(p, dict): - return None - e = p.get(module) - return e if isinstance(e, dict) else None - - -def may_access(user, module): - """May this account open `module` at all? Fail-closed on a migrated record.""" - if perms.is_admin(user): - return True - e = entry(user, module) - if e is not None: - return bool(e.get('access', True)) - if is_migrated(user): - # Migrated and undeclared = denied. This is the whole point of the marker. - return False - return perms.may_open(user, module) # legacy record: the old grant wall - - -def may_metrics(user, module): - """May this account build and receive METRIC columns — lookback measures — on `module`? - - ⭐⭐ W38-T19 — A CAPABILITY, NOT A SECOND SPELLING OF ACCESS, and the distinction is the - ticket. A rollup aggregates the CHILDREN a row is linked to; a Metric answers *"this number, - over this window"* against a governed topic with no relation at all (CLAUDE.md standing rule - 9). So it reads the book behind the rows rather than the rows: an account can be exactly the - right person to see a customer list and the wrong one to mint 12-month revenue over it. Two - decisions, two controls. - - ⛔ ABSENCE GRANTS, AND THE ASYMMETRY WITH `may_access` IS DELIBERATE RATHER THAN AN - OVERSIGHT TO TIDY. `may_access` reads migrated-and-undeclared as DENY, which is right there - because every save writes an entry for every governed database — absence means an - administrator decided. No `metrics` KEY was STORABLE before this ticket, so every migrated - record in every tenant carries none, and reading that absence as DENY would revoke Metrics - for everybody on the day this shipped with nobody having decided anything. That is the exact - failure `routes_admin.get_perms` records twice already (the `ut_*` default and the surface - default), one field further down the same entry. **Only an explicit `metrics: false` - refuses.** - - ⚠ IT DOES NOT RE-ASK ACCESS. Every caller is already behind the access wall (`module_gate`, - `may_read`, `session.require`), and folding admission in here would give a denial two - possible causes with one answer — the shape that makes a permission bug take an afternoon. - """ - if perms.is_admin(user): - return True - e = entry(user, module) - return not (isinstance(e, dict) and e.get('metrics') is False) - - -def nav_may_open(user, key, st=None): - """May this account SEE `key` — in the nav, and at the route gate? The ADMISSION question. - - ⛔⛔ WHY THIS EXISTS: THE EDITOR'S DECISION REACHED THE READ DOOR AND NOTHING ELSE. Measured - on live 2026-08-18, on a real account. An administrator ticked *Odoo products* for Naomi in - Manage user and saved; the record stored `perms.product_data.access = True` and - `may_access()` agreed. **The database never appeared.** `perms.nav_pages` and - `deps.Session.require` both ask `perms.may_open` — the LEGACY `user['modules']` array — which - still read `['sales', 'customers', 'products']`, and `products` is the key of the ARCHIVED - *SKU* module, not of `product_data`. So `nav_pages` answered `['customer_data']` and the - route would have 403'd her even by URL. Two permission systems, the editor writing one and - every DOOR reading the other ([[two-permission-systems-one-armed]]). - - ⭐ THE ASYMMETRY IS THE WHOLE DESIGN, and it is not the same rule twice: - - 1. an admin sees everything (break-glass, as everywhere else); - 2. an EXPLICIT `access: false` DENIES, on any key — an administrator unticking a box must - take the row off the nav, and before this it did not; - 3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` — creator, admin or - a `core.shares` grant — and **an `access: true` entry may NEVER widen past it**. The - editor must not become a way to hand somebody another user's private table; - 4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the - editor IS the authority: `_clean_perms` writes an entry for every governed topic on - every save, so an entry means an administrator decided; - 5. anything else — a module this editor does not govern — falls through to - `perms.may_open`, UNCHANGED. - - ⛔ LEG 5 IS LOAD-BEARING AND IT IS WHY `may_access` COULD NOT SIMPLY BE CALLED HERE. - `may_access` reads migrated-and-undeclared as DENY, which is correct for a governed topic and - catastrophic for the nav: Naomi's block declares the ten governed keys and nothing else, so - `sales` would have gone from visible to denied — an outage dressed as a permission fix. - """ - if perms.is_admin(user): - return True - e = entry(user, key) - if e is not None and not bool(e.get('access', True)): - return False - if str(key or '').startswith(_ut().KEY_PREFIX): - return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) - if e is not None: - return bool(e.get('access', True)) - return perms.may_open(user, key) - - -def assistant_entry(user, module): - """The explicit database grant an Assistant snapshot may rely on, else ``None``. - - The interactive application retains an admin break-glass path and a temporary legacy-grant - compatibility path. Neither is an answer at the Assistant's app-stored data boundary: - that reader must be able to name the migrated grant that admitted a database. In particular, - a store-outage admin identity with no ``perms`` document is not an unresolved permission that - may be widened into data access. - """ - e = entry(user, module) - if (not is_migrated(user) or not isinstance(e, dict) - or not bool(e.get('access', True)) or not may_access(user, module)): - return None - return e - - -# ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── -#: ⭐⭐ W38-T16 — THE MARKER THAT SAYS "THIS COLUMN IS GOVERNED BY A GRANT", stamped once by the -#: door that creates a shared column (`routes_tables.patch_shared_cell`) and never flipped by any -#: door afterwards. It is the `perms_v` idiom one object down, and it is here for the reason that -#: marker exists: **absence must not be read as a decision.** -#: -#: ⛔⛔ WHY A MARKER AND NOT "DOES A GRANT RECORD EXIST". Every tenant-wide column shipped before -#: this ticket carries no `field` grant record, because none was STORABLE — `shares.KINDS` had -#: three members. Reading that absence as "granted to nobody" would blank every existing shared -#: column for every account the moment this arms, which is not a permission fix, it is an outage -#: (`may_read` leg 3 carries the same argument for `ut_*` keys, in the same words). -#: -#: ⛔⛔ AND IT IS WHAT MAKES THE WALL FAIL **CLOSED**. The polarity here is the opposite of every -#: other grant check in this repo: a field share is a GRANT, so "no grant" has to mean HIDDEN or -#: the wall does nothing — but "no grant record" also describes a legacy column and a store that -#: could not be read. The marker separates the three: `granted` on the column means an explicit -#: decision was taken, so an unreadable registry hides it; no marker means legacy, so nothing -#: changes. Without it, one unreadable read of `object_shares` would publish every governed -#: column to the whole tenant, silently ([[a-guard-for-the-dangerous-case]]). -#: -#: ⚠ IT IS NOT `shared_overlay`'s `"shared": True` AND MUST NOT BE CONFUSED WITH IT. That flag is -#: written and never read (D-414) — `is_shared` is a dict-membership test — so it is not evidence -#: of anything. This one is read HERE, on every assembly, and the only writer is the create door. -FIELD_GRANT_MARK = 'granted' - - -def granted_field_keys(fields): - """Every column in `fields` that declares itself GOVERNED by a `shares` grant. - - ⭐ THE CHEAP HALF OF THE WALL, AND IT IS WHY THE WALL COSTS NOTHING FOR ALMOST EVERYBODY. It - is a scan of dicts already in memory, so a database with no governed column reaches no store - at all and `hidden_keys` behaves exactly as it did before this ticket. The registry is only - opened once this answers non-empty. - """ - return {str(f['key']) for f in (fields or ()) - if isinstance(f, dict) and f.get('key') - and f.get(FIELD_GRANT_MARK) is True} - - -def field_grant_hidden(user, table_key, fields, st=None): - """The governed columns of `table_key` this principal holds NO grant on — C1's per-FIELD wall. - - ⭐⭐ W38-T16 / R7 / R8 — THE THIRD WALL, AND IT COMPOSES ALONGSIDE THE OTHER TWO RATHER THAN - INSIDE THEM. `may_open` answers *IF* you reach a database; `may_read`'s stored `access: false` - overlay may REVOKE one; this answers *WHICH COLUMNS* of it you receive. It is deliberately not - threaded through that overlay: the overlay is **deny-only** by its own docstring (*"it may - revoke a database the wall below would admit, and it may never grant one that wall refuses"*) - and a field share is a GRANT — the widening direction. Merging them would give the codebase a - second idea of who grants what, which is the failure `may_open`'s own note is the record of. - It composes the way `may_open`'s grant leg does: additively, last, fail-closed. - - ⛔ AN ADMIN IS NOT WALLED (break-glass, as everywhere else) and neither is the column's OWNER — - `shares.role_for` answers `'owner'` for the creator, so a person cannot lose their own column - by forgetting to share it with themselves. - - ⚠ `st` IS THE TENANT HANDLE AND ITS ABSENCE IS SAFE HERE, unlike everywhere else. A caller - that cannot lend one reads the module-default bucket; on any tenant but #0 that finds no - grant, and no grant on a MARKED column means HIDDEN. So a door that has not learned to thread - `st` under-shares rather than over-shares, and the symptom is a grantee who cannot see their - column — visible, reportable, and the opposite of a leak. - """ - if perms.is_admin(user): - return set() - marked = granted_field_keys(fields) - if not marked: - return set() - uname = str((user or {}).get('username') or '').strip().lower() - try: - import core.shares as shares - except Exception: # noqa: BLE001 - return set(marked) - if not uname: - # No principal, and a marked column is an explicit decision: nobody is not somebody. - return set(marked) - # ⭐ THE CREATOR IS READ OFF THE COLUMN, NOT OUT OF THE REGISTRY, AND THAT IS NOT A SECOND - # AUTHORITY. `createdBy` is ALREADY what decides who may DELETE a shared column (R8 / D-172, - # `routes_tables.delete_shared_field`) and who may CLAIM it (`routes_shares._owns_object`); - # asking the same field here keeps one answer to "whose column is this" across all three. - # ⛔ IT IS ALSO THE ONLY THING THAT SURVIVES A CLAIM THAT NEVER LANDED. The create door writes - # the definition first and the grant record second, on purpose — so the window where a column - # is marked and unclaimed exists, and without this line its own author would be walled out of - # the column they just made, permanently and with no way to fix it but an admin. - mine = {str(f['key']) for f in (fields or ()) - if isinstance(f, dict) and f.get('key') and str(f['key']) in marked - and str(f.get('createdBy') or '').strip().lower() == uname} - hide = set() - for key in marked - mine: - try: - oid = shares.field_oid(table_key, key) - role = shares.role_for('field', oid, uname, is_admin=False, st=st) - except Exception: # noqa: BLE001 - role = None - if role is None: - hide.add(key) - return hide - - -def _measure_bound_keys(hidden, fields): - """Every column BOUND to a measure whose `measure_` pseudo-field is in `hidden`. - - ⭐ THE PREFIX IS IMPORTED, NEVER SPELLED. `aios_grid.MEASURE_FIELD_PREFIX` is the one - constant the client's `createField`, the host's `clean_measure_field` and now this wall all - key off; a literal "measure_" here would be the third copy, and the first to drift - [[constant-two-features-share]]. The import is lazy and function-local, which is the - established shape in this layer (`core/grid_events.py`, `core/user_tables.py` both do it) and - keeps `core` from pulling the grid module at import time. - - ⚠ CHEAP FIRST. Called only once `hidden` is already non-empty, and it returns on an empty - `wanted` before touching `fields` — so a wall with no metric tick costs one set - comprehension over a handful of strings, on a function that runs at every grid door. - - ⛔ A FAILED IMPORT HIDES NOTHING EXTRA RATHER THAN TAKING THE DOOR DOWN, matching - `field_grant_hidden`'s treatment of an unreachable `core.shares`. The direction is stated - because it is the unsafe one: this leg only ever WIDENS the hidden set, so losing it - under-hides — visible, reportable, and the symptom is a metric column that should have - been walled, not a database that will not open. - """ - try: - import aios_grid as _agm - prefix = _agm.MEASURE_FIELD_PREFIX - except Exception: # noqa: BLE001 - return set() - wanted = {h[len(prefix):] for h in hidden - if isinstance(h, str) and h.startswith(prefix) and len(h) > len(prefix)} - if not wanted: - return set() - out = set() - for f in fields or (): - if not isinstance(f, dict) or not f.get('key'): - continue - spec = f.get('measure') - if isinstance(spec, dict) and str(spec.get('key') or '') in wanted: - out.add(str(f['key'])) - return out - - -def hidden_keys(user, module, fields, st=None): - """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5). - - ⛔ WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER - (`formulaEngine.ts`, injected by `computedRows`) from `{ref}`s into other columns, and a - measure column's value arrives precomputed in `derived`. So hiding field X has exactly three - possible outcomes and only one of them is coherent: - - strip X, keep formulas → every formula over X computes blank or wrong, silently - keep X's value for them → X has leaked, wearing a formula's name - strip X AND its dependents→ the only honest answer - - So a hidden field drags every formula that references it — and every formula that references - THAT formula, hence the fixpoint loop — out of the payload with it. - - ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a - graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate - anyway; without it a self-referential pair would spin here. - - ⭐⭐ W36-T21 — AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this - closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup - names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that - link points at — so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices - this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns - what the hidden link contains, in aggregate; the three outcomes are exactly the three the - formula argument above enumerates, and only "strip both" is coherent. Verified against the - real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there - is either `{'link': , 'field': }` - or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes - and `rollup.field` is deliberately not treated as one — it names another database's column, - which has its own wall. - """ - if perms.is_admin(user): - return frozenset() - # ⭐⭐ W38-T16 — TWO SOURCES OF HIDING, ONE CLOSURE, AND THAT UNION IS THE WHOLE INTEGRATION. - # - # The administrator's `hiddenFields` and a field's own grant answer different questions and - # both end in the same place: a key this reader may not receive. Seeded together HERE, before - # the fixpoint, so the transitive argument above covers the new source unchanged — a formula - # (or a rollup) over a column this reader was not granted comes out with it, or the value - # leaks wearing the derived column's name. - # - # ⛔ AND THIS FUNCTION IS THE INSERTION POINT RATHER THAN `_scoped`, WHICH IS WHAT THE TICKET - # ASSUMED. `_scoped` is C1's evaluator and reaches C1's two doors; **the product reads through - # neither of them.** Every grid door calls THIS: `routes_customers:196`, `routes_products:345`, - # `routes_odoo_tables:920/1119`, `routes_tables._ut_hidden/_ut_field_wall`, `routes_slack:190`, - # `routes_grid:57/69/787` — and `grid_events` walls WRITES off the same set through - # `EventCtx.hidden_keys`. One evaluator, every door, both wires, read and write. - hidden = set(field_grant_hidden(user, module, fields, st=st)) - e = entry(user, module) - if e: - hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k} - if not hidden: - return frozenset() - # ⭐⭐ W40-T04 / I13 — A THIRD SOURCE, SEEDED BEFORE THE FIXPOINT FOR THE SAME REASON THE - # OTHER TWO ARE, AND IT IS WHAT MAKES THE NEW CHECKBOX ENFORCE ANYTHING. - # - # The permission editor now offers one `measure_`-namespaced PSEUDO-field per bound measure - # (`routes_admin._metric_fields`), so an administrator can tick "Metric - Gross margin $" and - # the wall stores `hiddenFields: ['measure_margin']`. But the COLUMNS that carry that number - # are keyed `measure__` by `CustomerGrid.createField` — `measure_margin_a1b2`, - # not `measure_margin` — and the seeding above compares KEYS. Without this line the box - # ticks, the record saves, every gate stays green and the reader keeps every gross-margin - # column on the grid. I13's own words are *"so a user can check the ones the permissioning is - # LIMITED TO"*; a control that limits nothing is the failure - # `verify_ui.py::metrics_toggle_has_a_mount_site` exists because of. - # - # ⭐ SO THE JOIN IS ON THE BINDING, NOT THE KEY: a hidden `measure_` hides every column - # whose `measure.key` IS ``, which is the same spec `aios_grid.clean_measure_field` - # wrote and `measure_fields_of` reads. One measure, however many windows a user minted over - # it, all walled by one tick. - # - # ⭐ ADDITIVE, NEVER SUBSTITUTIVE. The pseudo-key stays in `hidden` on its own account, so a - # real column that happens to be keyed exactly `measure_margin` is hidden exactly as before - # and nothing depends on the pseudo-field existing. - # - # ⛔ AND IT GOES HERE, ABOVE THE FIXPOINT, so the transitive argument this docstring already - # makes covers it unchanged: a FORMULA over a walled metric column, or a ROLLUP whose link is - # one, comes out with it. Seeded after the loop it would leak through the derived column, - # which is outcome two of the three the docstring enumerates. - hidden |= _measure_bound_keys(hidden, fields) - - refs = {} - for f in fields or (): - if not isinstance(f, dict) or not f.get('key'): - continue - expr = f.get('formula') - if isinstance(expr, str) and expr: - refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()} - link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None - if isinstance(link, str) and link.strip(): - refs.setdefault(f['key'], set()).add(link.strip()) - - MAX_PASSES = 12 - for _ in range(MAX_PASSES): - grew = False - for key, deps in refs.items(): - if key not in hidden and deps & hidden: - hidden.add(key) - grew = True - if not grew: - break - return frozenset(hidden) - - -def visible_fields(fields, user, module, st=None): - """`fields` minus the hidden closure. Order preserved — the column order is the user's. - - ⚠ W38-T16 — `st` MUST MATCH WHAT ITS CALLER PASSED TO `hidden_keys`, and that is not a style - note. This recomputes the closure, and since the field-grant leg under-hides without a tenant - handle, a caller that lends one to `hidden_keys` and not to this would narrow the FIELD LIST - by MORE than it stripped from the ROWS — the value left sitting in the payload under a column - nobody can see, which is exactly the half-wall `strip_row`'s note exists to forbid. - """ - hide = hidden_keys(user, module, fields, st=st) - if not hide: - return list(fields or ()) - return [f for f in (fields or ()) - if not (isinstance(f, dict) and f.get('key') in hide)] - - -def assistant_visible_fields(fields, user, module): - """Visible closure under one explicit Assistant grant, including for an admin caller.""" - e = assistant_entry(user, module) - if e is None: - return [] - # `hidden_keys` deliberately preserves the interactive admin break-glass behaviour. The - # Assistant uses its explicit grant instead, but shares the same transitive formula closure. - hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k} - if not hidden: - return list(fields or ()) - # ⛔⛔ THE SAME LEAK ONE DOOR OVER, AND IT IS THE SAME `hiddenFields`. `assistant_entry` - # returns the very dict `entry()` does, so once the permission editor can store - # `measure_margin` the Assistant's grant carries it too — and comparing KEYS would leave - # `measure_gross_profit_a1b2` in the snapshot this reader is handed. Seeded here for the - # identical reason and at the identical point as in `hidden_keys`: before the fixpoint, so a - # formula over a walled metric column comes out with it. - hidden |= _measure_bound_keys(hidden, fields) - - refs = {} - for field in fields or (): - if not isinstance(field, dict) or not field.get('key'): - continue - expr = field.get('formula') - if isinstance(expr, str) and expr: - refs[field['key']] = {match.strip() for match in _FORMULA_REF.findall(expr) - if match.strip()} - for _ in range(12): - grew = False - for key, deps in refs.items(): - if key not in hidden and deps & hidden: - hidden.add(key) - grew = True - if not grew: - break - return [field for field in (fields or ()) - if not (isinstance(field, dict) and field.get('key') in hidden)] - - -def strip_row(row, hide): - """Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run - per row: the field list and the row payload are two different wires, and stripping only the - first would leave the value sitting in the second where anyone can read it.""" - if not hide or not isinstance(row, dict): - return row - return {k: v for k, v in row.items() if k not in hide} - - -def visible_overlays(overlays, user, module, fields, st=None): - """The overlay CELL MAP minus every column this reader may not receive — D-427. - - ⛔⛔ THE THIRD WIRE, AND IT CARRIES THE RAW VALUE. `strip_row`'s own docstring makes the - argument for two wires: *"the field list and the row payload are two different wires, and - stripping only the first would leave the value sitting in the second where anyone can read - it."* There is a THIRD. `/workspace?scope=product` serves - `workspace["overlays"] = g["ws"].get("overlays")` verbatim — the persisted user/tenant - stratum, keyed `{row id: {field key: value}}` — and that assignment never asks who is - reading. So an administrator hides `first_cost`, the grid dutifully drops the column and the - cell, and the same number is still sitting in the workspace payload under the same key. The - wall holds on two wires out of three, which is not a wall. - - ⭐ WHY THIS LIVES HERE RATHER THAN AT THE DOORS. `hidden_keys` is already the ONE evaluator - every grid door calls, and the three leaking assignments are one line each in three files. - Putting the narrowing beside `strip_row` means the fix at each door is a single call to the - module that already owns the question, instead of a fourth place that decides what a reader - may see. A second idea of who hides what is the failure `may_open`'s own note records. - - ⭐ SAME ARGUMENT ORDER AS `visible_fields`, deliberately: a door that already narrows its - field list has the four values to hand, and `st` MUST be the same handle it passed there. - The field-grant leg under-hides without a tenant handle, so a door that lends one to - `visible_fields` and not to this would strip the COLUMN while leaving the overlay VALUE — - the half-wall this function exists to close, re-created by the fix for it. - - ⚠ NON-DESTRUCTIVE. A new dict is built rather than the caller's mutated, because the same - overlay object is read again by `rows_from_pool` on the assembly path; and the input is - returned untouched when nothing is hidden, so a database with no wall pays one set test. - """ - hide = hidden_keys(user, module, fields, st=st) - if not hide or not isinstance(overlays, dict): - return overlays - return {rid: strip_row(cells, hide) if isinstance(cells, dict) else cells - for rid, cells in overlays.items()} - - -# ── ROWS ───────────────────────────────────────────────────────────────────────────────────── -def _wall_leaf_keys(nodes): - """Every `colId` a filter tree names, at any depth. Groups carry `children`; leaves do not. - - ⚠ THE SAME WALK AS `routes_admin._leaf_col_ids`, and the duplication is deliberate rather - than lazy: that one runs at the ADMIN DOOR inside the API package, this one runs in `core` - on the request path, and `core` never imports up (see `platform/ARCHITECTURE.md`). The two - are three lines each and `verify_api` asserts they answer the same set on the same tree, so - a grammar change cannot land in one and not the other. That equality is the point: the door - refuses through one walk and the wall resolves through the other, and a wall the editor - accepts but the enforcement path cannot see is the defect this whole change is about. - - ⛔ AN `rhs` COLUMN IS NOT COLLECTED, MATCHING `_leaf_col_ids` AND FAILING CLOSED. A - column-to-column leaf whose RIGHT side is a user-generated column stays unenriched, so - `is_rule_active` finds that side outside the contract, calls the rule inactive, and strict - mode turns that into a DENY. That is the safe answer, and it is the same one this door gave - before the enrichment existed. `prune_filter_to_fields` does walk `rhs`, deliberately, and - the asymmetry is the direction each function fails in: deleting a leaf WIDENS a wall, so - that one must see every column a leaf names; refusing to enrich one only narrows. - """ - found = set() - for node in nodes or (): - if not isinstance(node, dict): - continue - if isinstance(node.get('children'), list): - found |= _wall_leaf_keys(node['children']) - elif node.get('colId'): - found.add(str(node['colId'])) - return found - - -def _enrich_for_wall(tree, rows, fields, module, st): - """`(rows, fields)` — the same wall inputs, plus any USER-GENERATED column this wall names - that the SERVER can actually answer. Owner I16, the half `ut_*` got for free. - - ⭐⭐ THE DEFECT. On the two Odoo topic grids the wall runs as - `apply_row_scope(rows, user, MODULE, )` over PRE-OVERLAY rows, so a - column a user created is neither declared in that field list nor present on the row — and - `filter_eval.permits` DENIES every leaf it cannot answer. Measured four ways on the - integrated head, one leaf `{colId: , op: eq, value: West}`, two rows: - - static contract + pre-overlay rows (THE LIVE CALL SITE) -> [] every row denied - column declared + rows carrying the value -> ['1'] correct - column declared + pre-overlay rows -> [] - a DECLARED odoo column, same shape -> ['1'] the evaluator is fine - - So the administrator saves a rule, is told it saved, and that account opens an empty grid - with nothing on screen saying why. BOTH halves are needed and neither alone does anything, - which is why this function supplies both from one read. - - ⛔⛔ IT MAY ONLY EVER ADD THE ABILITY TO ANSWER — IT MUST NEVER ADMIT A ROW THE WALL WOULD - OTHERWISE REFUSE. That is why `wallable_overlay_keys` is defined as *the keys whose values - the very `cells` read below serves*, and not as "every overlay column". Merge a blank for a - key this store cannot really answer and `custom_x is not West` flips from denying every row - (undeclared leaf) to admitting every row (`'' != 'West'`), with `is empty` doing the same — - a silent widening wearing the shape of a fix. The set and the values come from ONE stratum - so they cannot disagree about what is answerable. - - ⛔ AND THE WHOLE THING IS WRAPPED FAIL-CLOSED. On ANY failure the caller's own `rows` and - `fields` come back untouched and `permits` denies exactly as it does today. A wall that - cannot be enriched is a wall that keeps refusing, never one that opens. - - ⚠ THE ORDER IS A COST DECISION, NOT A STYLE ONE. `missing` is answered from the tree and the - field list already in memory, and an empty `missing` returns BEFORE `wallable_overlay_keys` - is called — so a wall naming only declared columns (every wall that exists today) pays one - set difference and reaches no store at all. - """ - try: - from harness import filter_eval as fe - # ⛔ `tree_parts`, NEVER `tree['nodes']`. A `FilterTree` is a PAIR and a BARE LIST is also - # a legal shape (C-PERM amendment 2); a second reader of it here would answer "no leaves" - # for a wall the evaluator one line down reads perfectly well. - nodes, _conj = fe.tree_parts(tree) - leaves = _wall_leaf_keys(nodes) - if not leaves: - return rows, fields, frozenset() - declared = {str(f['key']) for f in (fields or ()) - if isinstance(f, dict) and f.get('key')} - missing = leaves - declared - if not missing: - return rows, fields, frozenset() # nothing to add — the common case, and it is free - resolvable = missing - if not resolvable: # unreachable as written: `missing` is non-empty three lines up. - # Kept as the SHAPE of the guard, because `resolvable` is narrowed AGAIN below once - # the snapshot says what is actually answerable, and that narrowing can empty it. - # A wall naming a column NOTHING here can answer still denies, which is the whole - # point of `permits`. Enrichment is not a licence to ignore an unanswerable leaf. - return rows, fields, frozenset() - import core.shared_overlay as so - import core.view_templates as vt - ws_key = vt.workspace_key(str(module or '')) - if not ws_key: - return rows, fields, frozenset() - source = list(rows or ()) - # ⚠ THE ROW'S CELL KEY IS DERIVED BY `shared_overlay`'s OWN RULE, `str(int(pid))`, and - # not by `str(pid)`. That is the one coercion the stratum has (pids are ints in the grid - # and strings in JSON), so a second spelling here would look up `'1.0'` in a document - # keyed `'1'` and merge a blank over a value that is right there. It also RAISES on a - # pid-less or non-numeric row: a `cells` call carrying one would fail the whole request - # CLOSED, i.e. disable the fix silently for every other row rather than going red, so - # such a row is simply given the blank instead. - keys = {} - for i, r in enumerate(source): - if not isinstance(r, dict): - continue - try: - keys[i] = str(int(r.get('pid'))) - except (TypeError, ValueError): - continue - # ⭐⭐ ONE READ FOR THE WHOLE PAGE, AND FOR BOTH HALVES OF THE QUESTION. `snapshot` - # returns the shared stratum's SCHEMA and its CELLS from a single `st.get`, and it - # refuses an "everything" read by signature, so the row set handed in IS the bound. - # - # ⛔⛔ THE TWO HALVES MUST COME FROM ONE SNAPSHOT, AND THIS USED TO BE TWO READS. A - # wave-40 adversarial probe drove the gap: with the second read returning an emptied - # `cells`, a blank is merged for a key the store cannot really serve, and - # `custom_x is not West` flips from denying every row to ADMITTING a row whose true - # value IS West. `is empty` does the same. Narrow under today's cache-first store and - # structurally real under a threaded server, so it is closed by construction rather - # than by being unlikely. - _defs, cells = so.snapshot(ws_key, list(keys.values()), st=st) - wallable = wallable_overlay_keys(module, st=st, defs=_defs) - resolvable = resolvable & set(wallable) - if not resolvable: - return rows, fields, frozenset() - merged, denied = [], [] - for i, r in enumerate(source): - if not isinstance(r, dict): - merged.append(r) - continue - if i not in keys: - # ⛔⛔ A ROW WHOSE `pid` DOES NOT RESOLVE GETS NOTHING MERGED, AND THAT IS THE - # WHOLE POINT. `keys` holds only indices whose pid survived `str(int(pid))`; for - # any other row the store was never asked, so its value is UNKNOWN -- a different - # thing from the legitimately blank cell a real pid with no stored value has. - # Merging `''` for it hands `is not X` and `is empty` a FABRICATION and treats it - # as ground truth: a wave-40 adversarial probe drove exactly that, admitting a - # no-pid row under `is not West`, and a non-numeric-pid row under `less than 5` - # on a numeric column -- both of which the un-enriched wall refuses. Leaving the - # key off keeps the leaf unanswerable, so `permits` denies. - # - # ⚠ Not reachable through either registered reader today: `customer_data` and - # `product_data` both emit integer Odoo ids. Fixed because the invariant this - # function states is UNCONDITIONAL, not because it happened to be reachable. - # ⛔ DENIED, not merely un-merged. Leaving the key off the row is NOT - # enough: `wall_fields` DECLARES the column, and `permits` reads a declared - # field whose key is absent from the row as BLANK -- the same fabrication one - # level down, and measured doing exactly that. The index is recorded and the - # door drops the row outright. - merged.append(r) - denied.append(i) - continue - row_cells = cells.get(keys[i]) or {} - # A COPY, never the caller's dict mutated. These rows are the pool the tenant - # runtime caches and hands to every other consumer on the request (the workspace, - # the cohorts, the measures); writing a wall's working value into them would put a - # column on a payload nobody asked for it on. - merged.append(dict(r, **{k: row_cells.get(k, '') for k in resolvable})) - return (merged, list(fields or ()) + [wallable[k] for k in resolvable], - frozenset(denied)) - except Exception: # noqa: BLE001 - return rows, fields, frozenset() - - -def apply_row_scope(rows, user, module, fields, ctx=None, st=None): - """The rows this account may receive: `filter_eval.permits` over the permanent filter. - - `permits`, never `matches` — an unanswerable permanent filter DENIES rather than being - ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes - the difference a leak rather than a preference. - - ⭐⭐ `st` IS OWNER I16's SECOND HALF AND IT IS OPTIONAL SO THE FIRST HALF CANNOT MOVE. - *"Permission Filters must be able to filter on user-generated Fields too."* With a tenant - handle this door can resolve a user-generated column the static contract does not declare - (see `_enrich_for_wall`); WITHOUT one it is byte-identical to the wall that shipped before - this parameter existed, which is what keeps a cold process — a gate, a worker, E's sandbox — - behaving exactly as it always has. - - ⛔ PASS IT AT EVERY DOOR THAT WALLS A TOPIC GRID, OR AT NONE OF THEM. One stored wall read - through a door that lends the handle and a door that does not is one rule with two meanings, - which is worse than a uniform refusal: `allowed_pids` is the WRITE wall and `grid_assembly` - the READ one, and a user who may PATCH a row they cannot SEE is the hole - `allowed_pids`' own docstring exists to close. - """ - if perms.is_admin(user): - return list(rows or ()) - e = entry(user, module) - tree = (e or {}).get('filter') - if not tree: - return list(rows or ()) - from harness import filter_eval as fe - if st is not None: - # ⛔⛔ THE ENRICHMENT ANSWERS THE WALL AND MUST NEVER REACH A CALLER. It merges a - # column's value onto a COPY of each row so `permits` can evaluate a leaf naming it; - # returning those copies hands every consumer a column the caller's own field list - # does not declare. Measured in wave-40 QA against `core/script_sandbox.py`, which - # passes `scoped_table()`'s rows straight to a user-authored script: - # - # scoped_fields() declared keys: ['dba'] - # scoped_table() returned rows : [{'pid': 2, 'dba': 'Fisch', - # 'custom_region_qa': 'TOP-SECRET-VALUE'}] - # - # reachable by any ordinary session through `POST /script-views/{id}/run`. And the - # field-grant hide could never have caught it: `field_grant_hidden` can only mark a - # key already present in the `fields` it is handed, and this key never is. - # - # ⭐ So the decision is made on the enriched copy and the ORIGINAL row is what - # survives. `_enrich_for_wall` returns one entry per input row, in order, which is - # what makes the pairing sound; the copy is ONLY ever an argument to `permits`. - # ⚠ A length disagreement means the enrichment did not do what it promises, so the - # fall-through is the UN-enriched wall, which denies. Never the enriched rows. - # ⛔ MATERIALISED BEFORE THE ENRICHMENT SEES IT. `rows` may be any iterable, and - # `_enrich_for_wall` has three early-return paths that hand it straight back -- so a - # GENERATOR reached `len(judged)` and raised `TypeError: object of type 'generator' has - # no len()` on the common wall (one naming only declared columns), or, when enrichment - # did run, left the outer generator exhausted and the length check comparing against - # zero. Both fail closed, but one is a crash and the other a silent empty answer. One - # `list()` removes the class. Found by a wave-40 adversarial probe. - source = list(rows or ()) - judged, wall_fields, denied = _enrich_for_wall(tree, source, fields, module, st) - if len(judged) == len(source): - return [orig for i, (orig, seen) in enumerate(zip(source, judged)) - if i not in denied and fe.permits(tree, seen, wall_fields, ctx)] - return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)] - - -def assistant_apply_row_scope(rows, user, module, fields, ctx=None, st=None): - """Apply an explicit Assistant grant's permanent filter without an admin bypass. - - `st` carries the same meaning it does on `apply_row_scope`, for the same reason: an Assistant - grant is stored by the same editor, against the same vocabulary, and a wall that means one - thing on the grid and another in the Analyst's answer is two walls. - """ - e = assistant_entry(user, module) - if e is None: - return [] - tree = e.get('filter') - if not tree: - return list(rows or ()) - from harness import filter_eval as fe - if st is not None: - # Same rule as `apply_row_scope`: judge on the copy, return the ORIGINAL. See the block - # there for the measured leak this prevents. - # ⛔ MATERIALISED BEFORE THE ENRICHMENT SEES IT. `rows` may be any iterable, and - # `_enrich_for_wall` has three early-return paths that hand it straight back -- so a - # GENERATOR reached `len(judged)` and raised `TypeError: object of type 'generator' has - # no len()` on the common wall (one naming only declared columns), or, when enrichment - # did run, left the outer generator exhausted and the length check comparing against - # zero. Both fail closed, but one is a crash and the other a silent empty answer. One - # `list()` removes the class. Found by a wave-40 adversarial probe. - source = list(rows or ()) - judged, wall_fields, denied = _enrich_for_wall(tree, source, fields, module, st) - if len(judged) == len(source): - return [orig for i, (orig, seen) in enumerate(zip(source, judged)) - if i not in denied and fe.permits(tree, seen, wall_fields, ctx)] - return [row for row in (rows or ()) if fe.permits(tree, row, fields, ctx)] - - -def validate_assistant_filter(tree, fields): - """Return a strict, detached Assistant filter tree or raise ``ValueError``. - - The display filter cleaner is intentionally permissive: it drops a stale column or leaves an - inactive condition alone so an old saved view can still open. An Assistant data reader cannot - inherit that behaviour — dropping a predicate turns a request for a subset into a wider read. - This validator therefore admits only visible field operands that the existing evaluator can - answer from one stored row. Cohort, measure and rank conditions need separate materialised - set resolvers and are refused here rather than guessed. - """ - if tree is None: - return None - from harness import filter_eval as fe - - by_key = {str(field.get('key')): field for field in (fields or ()) - if isinstance(field, dict) and field.get('key')} - if not by_key: - raise ValueError('assistant filter has no visible field contract') - - def _leaf(raw): - if not isinstance(raw, dict): - raise ValueError('assistant filter leaf must be an object') - allowed = {'id', 'colId', 'op', 'value', 'value2', 'rhs'} - if set(raw) - allowed: - raise ValueError('assistant filter carries an unsupported operand') - col = raw.get('colId') - op = raw.get('op') - if not isinstance(col, str) or col not in by_key: - raise ValueError('assistant filter names an unknown or hidden field') - if (not isinstance(op, str) or op not in fe.FILTER_OPS or op in fe.RANK_OPS - or op in {'between', 'within'} or col == fe.COHORT_FIELD): - raise ValueError('assistant filter uses an unsupported operator') - rhs = raw.get('rhs') - if rhs is not None: - if (not isinstance(rhs, dict) or rhs.get('kind') != 'field' - or set(rhs) - {'kind', 'colId'} - or not isinstance(rhs.get('colId'), str) - or rhs['colId'] not in by_key): - raise ValueError('assistant filter names an unknown or hidden right-hand field') - out = {name: raw[name] for name in ('id', 'colId', 'op', 'value', 'value2') - if name in raw} - if rhs is not None: - out['rhs'] = {'kind': rhs.get('kind'), 'colId': rhs['colId']} - if not fe.is_rule_active(out, by_key): - raise ValueError('assistant filter is inactive or unanswerable') - return out - - def _node(raw): - if not isinstance(raw, dict): - raise ValueError('assistant filter node must be an object') - if 'children' not in raw: - return _leaf(raw) - if set(raw) - {'conj', 'children'}: - raise ValueError('assistant filter group carries an unsupported operand') - children = raw.get('children') - if raw.get('conj') not in ('and', 'or') or not isinstance(children, list) or not children: - raise ValueError('assistant filter group must be a non-empty and/or group') - return {'conj': raw['conj'], 'children': [_node(child) for child in children]} - - if isinstance(tree, list): - if not tree: - raise ValueError('assistant filter list must not be empty') - return {'conj': 'and', 'nodes': [_node(node) for node in tree]} - if not isinstance(tree, dict) or set(tree) - {'conj', 'nodes'}: - raise ValueError('assistant filters must be a tree with nodes') - nodes = tree.get('nodes') - if tree.get('conj') not in ('and', 'or') or not isinstance(nodes, list) or not nodes: - raise ValueError('assistant filter tree must be a non-empty and/or tree') - return {'conj': tree['conj'], 'nodes': [_node(node) for node in nodes]} - - -# ── USER-GENERATED COLUMNS + THE FILTER CASCADE (W40-T05 / owner I16) ──────────────────────── -def user_generated_fields(module, st=None): - """Every USER-CREATED column of `module`, across EVERY stratum -- or `None` when that - cannot be established. - - ⭐⭐ OWNER I16 — *"Permission Filters must be able to filter on user-generated Fields too. - If the field is deleted, its permission filter goes with it."* The permission editor built - its pickers from the STATIC contract (`aios_grid.FIELDS`, the product JSON), so a column a - USER made was invisible to the admin choosing what to filter on. This is the half that finds - them; `prune_filter_to_fields` below is the half that lets one go. - - ⛔⛔ `None` IS NOT `[]`, AND THE DIFFERENCE IS A SILENTLY WIDENED WALL. `[]` means - "resolved: this database has no user columns". `None` means "the vocabulary could not be - read". The cascade prunes a filter leaf naming a column that is NOT in this list, so a - DEGRADED answer would DELETE a live permission rule the moment the store was busy — - permanently, silently, and in the widening direction. Every unresolvable path therefore - answers `None` and every caller declines to prune on it. This is the GET/PUT skew - `routes_admin._clean_perms`' metric-tick note records one door over, except that there the - failure mode was a REFUSAL and here it would be a REVOCATION. - - ⚠ THE UNION OF EVERY STRATUM, DELIBERATELY. `_table_workspace` is - `{username: {views, fields, overlays}, '__shared__': {...}}` and a column lives in exactly - one of them. `_module_fields`' own docstring says this list is *"the admin choosing what to - hide"* and the SUBJECT USER IS NOT A PARAMETER OF IT, so a per-user read would make a column - unpickable for the very user who owns it. - - ⛔⛔ AND THE TENANT-WIDE STRATUM MOVED OUT OF THAT DOCUMENT, WHICH IS HOW "EVERY STRATUM" - STOPPED BEING TRUE. `core/shared_overlay.py`'s own RESIDENCY note says it: a SHARED column - lives in `_table_workspace__shared`, *"its own bucket, beside the per-user one — never - a `__shared__` member"*, and `field_permissions.promote_field` POPS the definition out of - the creator's stratum once it is promoted. So the loop below, reading one document, saw - exactly the columns nobody had shared. Measured on the live tenant: 21 of the 22 custom - columns on these two grids carry `source: "overlay", shared: true` — i.e. the function - returned the ONE column it was least useful for. - - ⛔ THE CONSEQUENCE WAS NOT A MISSING PICKER ROW, IT WAS A SILENT REVOCATION. This list is - also `routes_admin._prunable_vocabulary`'s answer, and `prune_filter_to_fields` DELETES a - leaf naming a column outside it. A wall stored against a shared column would therefore have - had its leaf pruned on the next read of the record — permanently, silently, and in the - widening direction, which is the exact failure the `None`-is-not-`[]` note above exists to - prevent, arriving through the door this function opens. - - ⚠ A SHARED READ THAT RAISES ANSWERS `None`, LIKE THE PER-USER ONE; an ABSENT shared bucket - is "nothing has been shared here yet" and does NOT poison the answer. Both arms are - deliberate: the first keeps the vocabulary honest under contention, and the second is what - stops a tenant that has never shared a column from losing the per-user half of I16. - - ⛔ THE BUCKET NAME IS `view_templates.workspace_key`, NEVER SPELLED. `customer_data`'s bucket - is `customer_table_workspace` — the module key is NOT the storage key — and `_WS_KEYS` is - already the one place that mapping lives ([[one-question-two-normalizers]]). - - ⛔ THE SHAPE COMES FROM `aios_grid.fields_from_workspace`, NEVER HAND-BUILT. That is the - function the GRID overlays a saved stratum with, so a column reaches the permission editor - described exactly as the user sees it — including `filterable: False` on a `measure_` column, - whose value is host-computed into `derived` and never sits on a row. A hand-shaped dict here - would be a second idea of what a field is, and the first thing it would lose is that flag. - `fields_base=[]` makes the base loop a no-op, so what comes back is the SAVED stratum alone. - - ⚠ `scope_key=None` DROPS A COHORT-SCOPED COLUMN, AND THAT IS THE ANSWER RATHER THAN A GAP. - `grid_events` writes `scope: 'cohort'` on a column created from the Cohort surface, and such - a column is not on the Customer grid's rows at all. Offering it to the permanent filter would - admit a leaf that can only ever DENY every row — the exact reason `_metric_fields` marks a - metric pseudo-field unfilterable. The narrow read is the honest one here. - """ - if st is None: - return None - try: - import aios_grid - import core.shared_overlay as shared_overlay - import core.view_templates as view_templates - except Exception: # noqa: BLE001 - return None - bucket = view_templates.workspace_key(str(module or '')) - if not bucket: - return None # a SURFACE, or nothing this layer knows as a table - try: - doc = st.get(bucket) - except Exception: # noqa: BLE001 - return None - if not isinstance(doc, dict): - # ⛔ AN ABSENT BUCKET IS `None`, NOT `[]`. A store that cannot answer and a database - # nobody has opened are indistinguishable from here, and only one of them is safe to - # prune against. Declining costs nothing real: a workspace with no bucket has no column - # to have deleted either. - return None - # ⛔ READ THROUGH `st` RATHER THAN `shared_overlay.fields()`, AND ONLY THE NAME COMES FROM - # THAT MODULE. `shared_overlay._read` swallows every exception and answers `{}`, so an - # unreadable store would be indistinguishable from "nothing is shared" — which is precisely - # the `None`-collapsed-into-`[]` this function refuses everywhere else. `bucket()` is still - # the ONE spelling of the key, so there is no second naming convention to get wrong. - try: - shared_doc = st.get(shared_overlay.bucket(bucket)) - except Exception: # noqa: BLE001 - return None - strata = list(doc.values()) - if isinstance(shared_doc, dict): - strata.append(shared_doc) - out, seen = [], set() - for blob in strata: - if not isinstance(blob, dict): - continue - saved = blob.get('fields') - if not isinstance(saved, dict) or not saved: - continue - try: - got = aios_grid.fields_from_workspace({'fields': saved}, fields_base=[]) - except Exception: # noqa: BLE001 - # One malformed stratum must not make the whole vocabulary unresolvable — that would - # turn a bad saved field into a frozen cascade. Skip it; the other strata still count. - continue - for f in got or (): - key = f.get('key') if isinstance(f, dict) else None - if not key or key in seen: - continue - seen.add(key) - out.append(f) - return out - - -def wallable_overlay_keys(module, st=None, defs=None): - """`{key: Field}` — the user-generated columns of `module` a PERMANENT FILTER can be - evaluated against on the server. Owner I16, narrowed to what is true rather than to what is - offered. - - ⭐⭐ THE DEFINITION IS "WHOSE VALUES `shared_overlay.cells` SERVES", AND EVERY OTHER PROPERTY - FOLLOWS FROM IT. `_enrich_for_wall` merges these keys onto the rows from exactly one read of - exactly this bucket, so defining the set any other way would let it name a column whose value - the merge cannot supply — and a blank merged for a column the store cannot really answer - turns `is not X` and `is empty` from "denies every row" into "admits every row". The set and - the values therefore come from ONE stratum, by construction rather than by care. - - ⛔ SO A `formula`, `created_time` OR `measure_*` COLUMN CAN NEVER BE IN HERE, AND IT IS - `aios_grid.fields_from_workspace` THAT SAYS SO RATHER THAN A LIST OF TYPE NAMES. That is the - normaliser the GRID itself overlays a saved stratum with: it emits the read-only user pair - and the measure columns as `source: 'odoo', derived: True` (their values are computed in the - BROWSER, or from the measure catalogue, and never sit on a stored row) and an editable - overlay column as `source: 'overlay'` with no `derived` at all. Testing the two flags is - testing the grid's own declaration; a hand-built type list here would be a second idea of - what a field is, and the first thing it would lose is the next read-only kind somebody adds. - - ⛔ AND A PER-USER PRIVATE OVERLAY COLUMN IS OUT TOO, which is the part that looks like a gap - and is not. Its values ARE server-readable (`[username]['overlays']`), but they are - readable only for the SUBJECT of the wall — an account that may edit that column freely, and - would therefore be one cell edit away from walking out of its own permission wall. The admin - who wrote the rule cannot even see the values. `routes_admin._row_wall_blind_keys` keeps - refusing those at the write door, where a person is present to choose a shared column - instead. - - ⚠ `{}` ON ANY FAILURE, NEVER `None` AND NEVER A PARTIAL SET. This is read by a wall, and the - only safe degraded answer for a wall is "I can answer nothing extra" — which leaves `permits` - denying an unresolvable leaf exactly as it does today. A half-resolved set could ADMIT a row, - which is the one direction this must never fail in. - - ⛔ AND IT ANSWERS `{}` FOR A `ut_*` DATABASE — DELIBERATELY, NOT BY ACCIDENT OF THE KEY. The - two topic grids name their shared stratum off the WORKSPACE key - (`customer_table_workspace__shared`), while a user table names its own off the TABLE key - (`routes_tables._ut_shared_fields` reads `shared_overlay.fields(table_key)`, i.e. - `ut_leads__shared`, not `ut_leads_table_workspace__shared`). Deriving from `workspace_key` - therefore finds nothing on a `ut_*` key, and that is the right answer TODAY rather than a - gap to paper over: a `ut_*` wall is validated by `routes_admin._module_fields` against the - database's own stored DEFINITION, which the enforcement path already declares and whose rows - already carry the values — so there is nothing for this to add, and the permission editor - cannot offer a `ut_*` shared column in the first place. Wiring that stratum in is a change - to what an admin may WALL ON, and it belongs with the picker change that offers it. - """ - if st is None: - return {} - try: - import aios_grid - import core.shared_overlay as shared_overlay - import core.view_templates as view_templates - - bucket = view_templates.workspace_key(str(module or '')) - if not bucket: - return {} - # ⭐ `defs` IS THE CALLER'S SNAPSHOT, AND PASSING IT IS NOT AN OPTIMISATION. - # `_enrich_for_wall` derives the VALUES from one read of this bucket and the - # answerable SET from this function; taking a second read here would let the - # two disagree, and a set that names a key the values cannot serve is exactly - # how a blank gets merged and `is not` flips from deny-all to admit-all. See - # `shared_overlay.snapshot`. - doc = ({'fields': dict(defs)} if defs is not None - else st.get(shared_overlay.bucket(bucket))) - saved = doc.get('fields') if isinstance(doc, dict) else None - if not isinstance(saved, dict) or not saved: - return {} - out = {} - for f in (aios_grid.fields_from_workspace({'fields': saved}, fields_base=[]) or ()): - if not isinstance(f, dict) or not f.get('key'): - continue - if f.get('source') != 'overlay' or f.get('derived'): - continue - out[str(f['key'])] = f - return out - except Exception: # noqa: BLE001 - return {} - - -def prune_filter_to_fields(tree, valid_keys): - """`(tree, dropped)` — `tree` with every leaf naming a column outside `valid_keys` removed. - - ⭐⭐ THE CASCADE OWNER I16 ASKS FOR: *"If the field is deleted, its permission filter goes - with it."* A stored permanent filter naming a column the database no longer has is not - ignored by the wall — `apply_row_scope` uses `permits`, which DENIES anything it cannot - answer — so a deleted column silently converts that account's grid to zero rows. Dropping - the leaf is what the owner accepted instead. - - ⛔⛔ THIS IS AN ADMIN-DOOR OPERATION AND IT MUST NEVER MOVE INTO THE ENFORCEMENT PATH. - `verify_perm_scope`'s row-scope leg asserts, in these words, *"a wall naming a DELETED column - denies every row (never ignored)"* — it puts `{'colId': 'ghost', ...}` in a stored filter and - calls `apply_row_scope` directly. If the wall started ignoring unknown leaves, that gate would - flip in the FAIL-OPEN direction and a real permission wall would quietly stop walling. So the - prune runs where a RECORD is read or written, and the wall keeps denying as the backstop for - anything the prune has not reached yet. - - ⛔ PER LEAF, WHERE `routes_admin._prune_to_module` IS ALL-OR-NOTHING PER GROUP, AND THE - DIFFERENCE IS DELIBERATE. That function folds a LEGACY wall into a module that may not be - able to evaluate it, where half a group is a rule nobody wrote. This one answers a different - question: one named column is GONE, and the owner's instruction is that its leaf goes with it - while the rest of the admin's rule stands. `done-when` says so in as many words -- *"leaving - the user's other filters intact"*. - - ⚠ AND THE WIDENING IS REAL, SO IT IS STATED RATHER THAN BURIED. Dropping a leaf from an `and` - group WIDENS that wall, and dropping the last leaf anywhere removes it altogether. That is - the direction the owner chose over deny-every-row; it is also why `valid_keys` must be a - RESOLVED vocabulary (see `user_generated_fields`) and never a degraded one. - - ⚠ A LEAF'S `rhs` COUNTS AS NAMING A COLUMN. A column-to-column comparison whose right side - was deleted is just as stale as one whose left side was, and `clean_filter_tree` would answer - it by dropping the `rhs` alone -- turning "revenue > forecast" into a comparison against a - literal, which is a DIFFERENT question wearing the original's shape. - - ⛔ AN EMPTIED TREE IS `None`, NEVER `{'conj': 'and', 'nodes': []}`. Measured: `permits` - returns True on an empty node list, so an empty-but-present tree admits every row -- while - `wall_declared` and `row_scope_applies` both read it as TRUTHY and answer that a wall applies. - A door would then build rows to filter them against nothing, and an editor would paint a rule - that is not there. `None` is the one shape all three agree about. - """ - valid = {str(k) for k in (valid_keys or ())} - if not valid: - # ⛔ A FLOOR, NOT AN OPTIMISATION. An empty vocabulary would prune EVERY leaf, which for a - # module whose schema momentarily failed to resolve is the whole wall gone in one read. - return tree, [] - dropped = [] - - def _keep(node): - if not isinstance(node, dict): - return None - kids = node.get('children') - if isinstance(kids, list): - surviving = [k for k in (_keep(k) for k in kids) if k is not None] - # An emptied GROUP carries no meaning once persisted -- the same rule - # `clean_filter_tree` applies to one it built. - return dict(node, children=surviving) if surviving else None - named = [node.get('colId')] - rhs = node.get('rhs') - if isinstance(rhs, dict) and rhs.get('kind') != 'stat' and rhs.get('colId') is not None: - named.append(rhs.get('colId')) - stale = [str(c) for c in named if c is not None and str(c) not in valid] - if stale: - dropped.extend(stale) - return None - return node - - if not isinstance(tree, dict): - return tree, [] - nodes = tree.get('nodes') - if not isinstance(nodes, list): - return tree, [] - kept = [n for n in (_keep(n) for n in nodes) if n is not None] - if not dropped: - return tree, [] # unchanged by construction, not merely equal - if not kept: - return None, sorted(set(dropped)) - return {'conj': 'or' if tree.get('conj') == 'or' else 'and', 'nodes': kept}, \ - sorted(set(dropped)) - - -# ── THE PUSHDOWN ───────────────────────────────────────────────────────────────────────────── -#: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly -#: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission -#: filter admitting Fisch admits {Fisch, Both} — a Both customer IS a Fisch customer. -_DBA_TEAM = ((frozenset({'fisch', 'both'}), 5), (frozenset({'royal', 'both'}), 6)) - -#: Only equality pushes down. `FILTER_OPS` is single-valued (there is no column-level "is any -#: of" — that vocabulary belongs to cohort leaves and is disjoint), so a multi-value BU -#: condition arrives as an OR GROUP of `eq` leaves, handled by `_group_dba_team` below. -_PUSHDOWN_OPS = frozenset({'eq'}) - -#: ⭐ THE MODULES WHOSE FIELD VOCABULARY CAN EXPRESS A BUSINESS UNIT — i.e. whose canonical -#: contract carries a `dba` column. R1's "BU access is just a permanent filter" holds only on -#: these; on every other topic there is no column to write the condition against, so a filter -#: literally cannot say it and the record's `bus` remains the only place the fact lives. -#: -#: `product_data` is the counter-example that made this constant necessary: 19 fields, no brand -#: column, and a product's BU is a property of WHOSE ORDERS built its revenue rather than of the -#: SKU. Without the fallback below, a Fisch-only account read the product catalogue with -#: Fisch+Royal money on every row — the amendment-3 defect, arriving through the door amendment 3 -#: was written to close. -#: -#: ⚠ A LIST THAT MIRRORS A JSON CONTRACT DRIFTS UNLESS SOMETHING CHECKS IT. `verify_api` asserts -#: membership here matches "this topic's contract has a `dba` field" for every governed module, so -#: a topic that grows or loses a brand column cannot silently keep the wrong rule. Named in this -#: module rather than derived from `aios_grid` on purpose: `perm_scope` is on the API's request -#: path and importing the grid stack to answer a two-element question is a cost per request. -BU_FILTERABLE_MODULES = frozenset({'customer_data'}) - - -def derive_pool_scope(user, module): - """`(team_id, agent)` the POOL must be BUILT with, derived from the permanent filter. - - ⛔ THIS EXISTS BECAUSE `team_id` SHAPES VALUES, NOT ROW MEMBERSHIP (C-PERM amendment 3). - `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, - LY, LTM) and into `cust._cadence_bulk`, so it decides what `rev`, `ly`, `ltm`, `aov`, - `est_missed` and the derived `status` MEAN. Enforce a BU purely as a post-filter and a - Fisch-only user keeps a correct-looking row LIST while every number on it silently becomes - Fisch+Royal — worst for `dba = Both` customers, who are exactly the ones a BU filter admits. - A pid-level reconciliation cannot see that; the values one can, and does. - - So the query-level pushdown SURVIVES — but as a DERIVATION OF the permanent filter rather - than a second wall beside it, which is what keeps R1's "BU access is just a filter" true at - the level the owner asked for it (one declaration, one UI, one engine). - - Pure, and recomputed per request rather than stored: a stored derivation drifts from the - filter it came from, and then two things disagree about what an account may see. - - Reads TOP-LEVEL AND-conjunction leaves ONLY. A leaf under `or` guarantees no narrowing — - `dba is Fisch OR revenue > 10` must not pin the pool to Fisch — so it never pushes down. - Anything not recognised here simply is not pushed down; `apply_row_scope` still applies the - whole tree, so the wall is unchanged either way. Belt AND braces, deliberately: the pushdown - is what makes the VALUES right, `permits()` is what makes the ROWS right. - - ⭐ THE `bus` FALLBACK, AND WHY IT IS NOT A HOLE IN AMENDMENT 4. On a topic outside - `BU_FILTERABLE_MODULES` there is no column a BU condition could be written against, so - "the filter pins no team" cannot mean "the admin chose consolidated" — it is the only answer - the filter language has. Resolving that silence as None returns the WIDER scope, which makes - the current code fail-OPEN on the values axis for exactly the topic that cannot argue back. - So the record's own `bus` answers instead, and the direction is what makes it safe: this can - only ever REPLACE None (both units) with a pinned single unit. It never widens, it never - touches `may_access`, and a `bus:'all'` account is unaffected because `scope_team_id` returns - None for it — which is every account in tenant #0's registry except the one this shipped for. - """ - if perms.is_admin(user): - return None, None - team_id, agent = _derive_from_filter(user, module) - if team_id is None and module not in BU_FILTERABLE_MODULES: - team_id = perms.scope_team_id(user) - return team_id, agent - - -def _derive_from_filter(user, module): - """`(team_id, agent)` the PERMANENT FILTER pins, before any fallback. Split out so the - fallback has exactly one place to apply — the three exits below all mean "the filter pinned - nothing", and a rule written at each of them is a rule that will one day be written at two.""" - e = entry(user, module) - tree = (e or {}).get('filter') - if not tree: - # Un-migrated records still answer through the legacy derivation, so the old wall keeps - # working until the migration has run. - if not is_migrated(user): - return perms.scope_team_id(user), perms.scope_agent(user) - return None, None - - from harness import filter_eval as fe - nodes, conj = fe.tree_parts(tree) - if conj == 'or': - return None, None - - team_id, agent = None, None - for n in nodes: - if not isinstance(n, dict): - continue - if isinstance(n.get('children'), list): - # A top-level OR GROUP under an AND root IS a guaranteed narrowing — every row must - # satisfy it — so it may push down, unlike a leaf under an OR ROOT (refused above). - # This is the shape a multi-value BU condition actually takes; see `_group_dba_team`. - tid = _group_dba_team(n) - if tid is not None: - team_id = tid if team_id in (None, tid) else None - continue - if n.get('op') not in _PUSHDOWN_OPS: - continue - col = n.get('colId') - raw = n.get('value') - if col == 'dba': - vals = {v.strip().lower() for v in str(raw or '').split(',') if v.strip()} - if not vals: - continue - for allowed, tid in _DBA_TEAM: - if vals <= allowed: - # Both BUs named = no narrowing to push; leave it to the post-filter. - team_id = tid if team_id in (None, tid) else None - break - elif col == 'agent': - v = str(raw or '').strip() - # A SET of agents cannot become the pool's single `agent_name`; the post-filter - # handles it. Only an unambiguous single value pushes down. - if v and ',' not in v: - agent = v - return team_id, agent - - -def _group_dba_team(group): - """The team a top-level `or` group pins, or None. - - Recognises ONLY the exact shape "every child is a `dba eq ` leaf" — the group the - condition builder emits for a multi-value BU condition, and the one `perm_migrate` writes. - Every OTHER group returns None and is left entirely to the post-filter: a group mixing `dba` - with another column, or containing a nested group, does not pin a BU on its own, and - guessing that it does would build the pool from the wrong book. Narrow by construction — - the pushdown may only ever be an OPTIMISATION of a constraint the filter already expresses. - """ - if group.get('conj') != 'or': - return None - children = group.get('children') or [] - if not children: - return None - vals = set() - for c in children: - if (not isinstance(c, dict) or isinstance(c.get('children'), list) - or c.get('colId') != 'dba' or c.get('op') != 'eq'): - return None - v = str(c.get('value') or '').strip().lower() - if not v: - return None - vals.add(v) - for allowed, tid in _DBA_TEAM: - if vals <= allowed: - return tid - return None - - -# ── C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ──────────────────────────────── -#: ⭐⭐ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same -#: permission logic, always"* — per-user field visibility AND row filtration on every database -#: carrying a unique id, whatever created it, with a NEW database inheriting it by construction -#: rather than by a list somebody maintains. -#: -#: ⛔ THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line -#: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic -#: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone — -#: creator, admin, or a `core.shares` grant — which is a BINARY door: you see all 31,418 rows of -#: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this -#: work in as many words (*"Arming `perm_scope` over `ut_*` … booked, not faked"*), and owner -#: item 11 is that booking coming due. -#: -#: ⚠ AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT -#: user-created. Ten of them (`ut_odoo_invoices`, `…_orders`, `…_agents`, `…_accounts`, `…_bills`, -#: `…_vendors`, `…_order_lines`, `…_gl_lines`, `…_customers`, `…_products`) are generated by the -#: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a -#: statement about origin**, and a wall keyed off it was reading a naming artefact as a security -#: boundary. -#: -#: ⛔⛔ THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and -#: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES -#: them — it calls `may_open`, it does not reimplement it — because merging them is how this -#: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20 -#: note). One resolver per question, asked in order. - - -class UnknownTable(LookupError): - """No database in this tenant answers to that key. - - ⛔ RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this - database is empty"* — indistinguishable from a real empty table, and the caller least able to - notice is the one that wanted rows. This repo has shipped that exact silent-empty answer - before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]). - """ - - -class Denied(PermissionError): - """This principal may not read this database at all. The IF question, answered by `may_read`.""" - - -class Unresolvable(RuntimeError): - """The rows exist and cannot be served under this call's constraints — R6's SECOND SENTENCE. - - ⭐ STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there - is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a - limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never - silently enforced as a short answer. Carries the same four keys - `routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight - to a client without a second vocabulary ([[one-question-two-normalizers]]). - """ - - def __init__(self, subject, effect, cause, recommendation): - self.subject, self.effect = subject, effect - self.cause, self.recommendation = cause, recommendation - super().__init__(f"{subject}: {effect}. {cause}. {recommendation}") - - def as_limit(self): - """The dict shape `routes_tables` puts in an assembly's `limits` list.""" - return {"subject": self.subject, "effect": self.effect, - "cause": self.cause, "recommendation": self.recommendation} - - -#: Row readers DECLARED by the app layer, keyed by EXACT database key. -#: `reader(table_key, user, st) -> (fields, rows)`. -#: -#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and -#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache -#: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables` -#: already uses for exactly this reason — `register_connected`, `register_read_through`, -#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being -#: interrogated by it."* -_ROW_SOURCES = {} - -#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. -#: `reader(table_key, field_keys, st) -> rows`. -_MIRROR_READER = None - - -def register_rows(reader, *table_keys): - """Declare who reads a NAMED database's rows. Returns the registered key set. - - ⚠ The return value is the registrar's own answer on purpose: a public function whose only - caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that - says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write - door's return keeps one construction site of the set instead of two. - """ - for key in table_keys: - k = str(key or '').strip() - if k: - _ROW_SOURCES[k] = reader - return frozenset(_ROW_SOURCES) - - -def register_mirror(reader): - """Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`).""" - global _MIRROR_READER - _MIRROR_READER = reader - return _MIRROR_READER is not None - - -#: ⛔ `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE -#: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE -#: list to read"* — and `register_rows` ALREADY returns exactly that, which is the same idiom -#: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through -#: the write door's return keeps ONE construction site of the set. A second accessor beside it is a -#: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py` -#: — the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by -#: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the -#: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered. - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def may_read(user, table_key, st=None): - """May this principal read `table_key` AT ALL — the IF question, on EVERY database. - - ⛔ COMPOSED, NOT RE-DERIVED, and the order is the whole rule: - - 1. an admin reads everything (break-glass — `deps._user_for` hands back a hardcoded master - dict on a store outage and it will never carry a `perms` block); - 2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner - item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall - below would admit, and it may never grant one that wall refuses; - 3. a `ut_*` database defers to `user_tables.may_open` — creator, admin, or a `core.shares` - grant — UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."* - 4. anything else is a registry topic and defers to `may_access` above. - - ⛔⛔ WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does. - `may_access` reads migrated-and-undeclared as DENY — correct for a topic, because - `routes_admin` writes an entry for every governable topic on every save. ⚠ NO `ut_*` ENTRY WAS - STORABLE AT ALL UNTIL W36-T22 — `_clean_perms` refused the key with `unenforced_module` — so - every record migrated before this wave carries no entry for any of them, and reading that - absence as a decision would revoke all ten keychain databases from every migrated account the - moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been - answering rather than the marker that has not, and it keeps being right AFTER the flag's - deletion: an admin who has never opened the editor for a database has still not decided - anything about it. - - ⛔⛔ PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a - stored record is keyed BY username in that bucket and does not carry one INSIDE it — only - `core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand - this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*` - database reads as DENIED for an account that can open all of them. It fails in the SAFE - direction and is silently wrong, which is the worst pair to debug — it cost two call sites in - one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage. - """ - if perms.is_admin(user): - return True - e = entry(user, table_key) - if e is not None and not bool(e.get('access', True)): - return False - key = str(table_key or '') - if key.startswith(_ut().KEY_PREFIX): - return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) - return may_access(user, table_key) - - -def wall_declared(user, table_key): - """Is a ROW or FIELD narrowing declared for this principal on this database? - - ⛔ THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a - stored `ut_*` wall would be INERT — *"the editor would say DENY, the table routes would keep - serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore - REFUSE for a principal this returns True for, rather than serve the whole database. False for - an admin (they bypass the wall entirely) and for any record with no entry, so a door asking - this pays nothing and changes nothing for everybody who has no wall. - """ - if perms.is_admin(user): - return False - e = entry(user, table_key) - if not e: - return False - return bool(e.get('filter')) or bool(e.get('hiddenFields')) - - -def row_scope_applies(user, table_key): - """Does a permanent ROW filter narrow this principal on this database? - - ⚠ `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows - it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its - whole point is that the pid set costs no row pass, and paying for one on every database - switch — for every account, walled or not — would undo W30-T30 to enforce a rule that applies - to almost nobody. Asked here rather than spelled out at the call site, so there is ONE - statement of when the row wall bites ([[one-question-two-normalizers]]). - """ - if perms.is_admin(user): - return False - return bool((entry(user, table_key) or {}).get('filter')) - - -def scoped_table(user, table_key, st=None, ctx=None): - """⭐⭐ CONTRACT C1 — the rows of ANY database, already field-stripped and row-filtered for - `user`. Registry topic or `ut_*`; there is no third kind and no per-database branch. - - rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database - rows = scoped_table(user, 'customer_data') # a registry topic - - `user` is a user RECORD (the dict `deps.Session.user` carries), not a username — the whole - wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller - that forgets the principal must not run, because the only thing a defaulted one could mean is - "unscoped", which is the widening direction. - - ⛔ FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the - right status instead of guessing: `UnknownTable` (no such database — never an empty list), - `Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is - why — standing rule 1's second sentence). - - ⚠ NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only - thing that stops it is a population that exceeds one materialisation window, and that arrives - as `Unresolvable` carrying its cause and a recommendation rather than as a short answer. - - ⭐ E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is - THE door rather than A door: everything a sandboxed script may read, it reads here, under the - CALLING user's scope (R5). - """ - _fields, rows = _scoped(user, table_key, st=st, ctx=ctx) - return rows - - -def scoped_fields(user, table_key, st=None): - """The COLUMNS of any database this principal may see — C1's other half. - - ⛔ IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the - field list and the row payload are two different wires, and narrowing one without the other - leaves the value sitting where anything can read it. A caller that must render a scoped table - needs both, and E cannot read a `ut_*` definition to learn its columns — the sandbox has no - second path to the store (W1). So both come from here, off one wall. - - ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a - registry topic it goes through the registered reader, which builds that topic's pool; the - pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. - """ - fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) - return fields - - -def _scoped(user, table_key, st=None, ctx=None, want_rows=True): - """`(fields, rows)` — ONE evaluator behind both public doors, so they cannot disagree.""" - key = str(table_key or '').strip() - if not key: - raise UnknownTable('a database key is required. This door will not guess which database ' - 'was meant') - if not may_read(user, key, st=st): - raise Denied(f"this account may not read '{key}'") - fields, rows = _read(key, user, st, want_rows) - # THE FIELD WALL — a TRANSITIVE closure, so hiding a column also hides every formula computed - # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference - # is the wrong shape here. - # ⭐ W38-T16 — `st` RIDES INTO THE WALL, not just into the read. The field-grant leg resolves - # against `object_shares` in THIS tenant's store; without the handle it would answer from the - # module default (tenant #0) and hide a grantee's own column on every other tenant. - hide = hidden_keys(user, key, fields, st=st) - if not want_rows: - return (visible_fields(fields, user, key, st=st) if hide else fields), [] - # THE ROW WALL — `permits()`, so a permanent filter this evaluator cannot answer DENIES - # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a - # permanent filter may name a column the reader is not allowed to SEE, and dropping the - # predicate would widen the read rather than narrow it. - # ⭐ OWNER I16 — `st` RIDES INTO THE ROW WALL TOO, and this door is the one that already had - # the handle and simply did not pass it down. The field wall two lines up has taken it since - # W38-T16 for the same reason: a wall resolved without a tenant handle answers from the - # module default, and here that means a user-generated column reads as unanswerable and - # denies every row. - rows = apply_row_scope(rows, user, key, fields, ctx, st=st) - if hide: - fields = visible_fields(fields, user, key, st=st) - rows = [strip_row(r, hide) for r in rows] - return fields, rows - - -def _read(table_key, user, st, want_rows=True): - """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" - reader = _ROW_SOURCES.get(table_key) - if reader is not None: - fields, rows = reader(table_key, user, st) - return list(fields or ()), list(rows or ()) - ut = _ut() - if not table_key.startswith(ut.KEY_PREFIX): - # ⛔ A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never - # imported the API layer this is the honest answer: nothing here can build that pool. - raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader " - f"is registered for it") - return _read_user_table(table_key, st, want_rows) - - -def _read_user_table(table_key, st, want_rows=True): - """core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately. - - ⭐ WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same - argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold - process — E's sandbox subprocess, a worker, a gate — that never imported an API route still - owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and - the sandbox is exactly such a process. - - ⚠ THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions - without the 28.6 MB of rows (D-213), which is every read this function makes when - `want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so - the materialised arm below takes the whole read explicitly. - """ - ut = _ut() - lent = ut.lend_defs(st) - defn = ut.get(table_key, st=lent) - if not defn: - raise UnknownTable(f"no database named '{table_key}'") - fields = [dict(f) for f in (defn.get('fields') or [])] - if not want_rows: - return fields, [] - if not ut.materialises(table_key, st=st, defn=defn): - # A read-through grid stores no rows here — they live in the mirror, and reading - # `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking, - # wrong, and silent. - if _MIRROR_READER is None: - raise Unresolvable( - subject='rows', effect='unreadable', - cause=(f"'{table_key}' is served read-through from the connector mirror and no " - f'mirror reader is registered in this process'), - recommendation=('call `perm_scope.register_mirror(...)` from the app layer before ' - 'reading a read-through database, or read it through the API')) - keys = {f['key'] for f in fields if f.get('key')} - return fields, list(_MIRROR_READER(table_key, keys, st) or ()) - whole = ut.get(table_key, st=st) - if whole is None: - # Deleted between the wall and here. The same refusal, not an empty table. - raise UnknownTable(f"no database named '{table_key}'") - field_keys = {f['key'] for f in fields if f.get('key')} - rows = [] - for rid, row in (whole.get('rows') or {}).items(): - if not str(rid).isdigit(): - continue - r = {k: v for k, v in (row or {}).items() if k in field_keys} - r['pid'] = int(rid) - rows.append(r) - rows.sort(key=lambda r: r['pid']) - return fields, rows +"""core/perm_scope.py — the permission WALL for table modules (wave 15, C-PERM). + +ONE place answers the three questions a restricted account raises on a grid surface: + + may_access(user, module) may they open it at all? + visible_fields(fields, u, mod) which COLUMNS may they receive? + apply_row_scope(rows, u, mod, …) which ROWS may they receive? + +plus one that exists only because of how the pool is built: + + derive_pool_scope(user, module) which (team_id, agent) must the pool be BUILT with? + +Both hosts call these — `aios-web/api` (`grid_assembly`) and `app.py` (`_table_grid`) — because +a wall that exists on one runtime and not the other is not a wall. `core/perms.py` stays what it +is (module GRANTS + the legacy BU derivation); this module is the row/field/pushdown layer that +sits on top, and it is deliberately a separate file so the legacy readers can keep their +semantics untouched while this one fails closed. + +──────────────────────────────────────────────────────────────────────────────────────────── +THE RECORD + + user['perms'] = {'': {'access': bool, + 'filter': {'conj'?: 'and'|'or', 'nodes': [...]} | None, + 'hiddenFields': ['', ...]}} + user['perms_v'] = 1 stamped by the migration and by every write + +`perms_v` is the EXPLICIT-RESOLUTION marker, and it is here because of `permissioning.md` +Part II gap #5: "`allowed_modules() is None` is fail-open by default … make 'resolved: +unrestricted' an explicit value so absence/uncertainty DENIES." The same class already shipped +twice in this codebase (`modules: []` and `bus: []` both read as UNRESTRICTED — +`routes_admin.py`'s own docstring documents both). So: + + * `perms_v` ABSENT → the record is UN-MIGRATED, and the LEGACY wall applies unchanged + (`core.perms` module grants + the `bus`/`agent` query scope). That is not fail-open: it is + today's real wall, and it bounds the rollout window to "until the migration runs". + * `perms_v == 1` and the module has NO entry → **DENY**. Absence now means what it says. + * `role == 'admin'` bypasses all of it — which is also what keeps BREAK-GLASS alive. + `deps._user_for` hands back a hardcoded master dict on a store outage + (`{'username':'admin','role':'admin','bus':'all','modules':'all'}`) that will never carry a + perms block; without this clause an explicit-marker scheme locks the owner out of their own + product at exactly the moment the store is broken. +""" +import re + +import core.perms as perms + +#: `aios_grid._FORMULA_REF`'s pattern, restated rather than imported: this module is imported by +#: the API's request path and `aios_grid` pulls in the whole grid stack. Same regex, one line, +#: and `verify_api` asserts the two agree so it cannot drift into a different grammar. +_FORMULA_REF = re.compile(r"\{([^{}]*)\}") + +PERMS_VERSION = 1 + + +def _rec(user): + return user if isinstance(user, dict) else {} + + +def is_migrated(user): + """True once this record carries an explicit resolution. See the module docstring.""" + return int(_rec(user).get('perms_v') or 0) >= PERMS_VERSION + + +def entry(user, module): + """This user's declared permissions for `module`, or None if nothing is declared. + + None is AMBIGUOUS on purpose and every caller must resolve it against `is_migrated`: + on a migrated record it means DENY, on a legacy one it means "ask the old wall". + """ + p = _rec(user).get('perms') + if not isinstance(p, dict): + return None + e = p.get(module) + return e if isinstance(e, dict) else None + + +def may_access(user, module): + """May this account open `module` at all? Fail-closed on a migrated record.""" + if perms.is_admin(user): + return True + e = entry(user, module) + if e is not None: + return bool(e.get('access', True)) + if is_migrated(user): + # Migrated and undeclared = denied. This is the whole point of the marker. + return False + return perms.may_open(user, module) # legacy record: the old grant wall + + +def may_metrics(user, module): + """May this account build and receive METRIC columns — lookback measures — on `module`? + + ⭐⭐ W38-T19 — A CAPABILITY, NOT A SECOND SPELLING OF ACCESS, and the distinction is the + ticket. A rollup aggregates the CHILDREN a row is linked to; a Metric answers *"this number, + over this window"* against a governed topic with no relation at all (CLAUDE.md standing rule + 9). So it reads the book behind the rows rather than the rows: an account can be exactly the + right person to see a customer list and the wrong one to mint 12-month revenue over it. Two + decisions, two controls. + + ⛔ ABSENCE GRANTS, AND THE ASYMMETRY WITH `may_access` IS DELIBERATE RATHER THAN AN + OVERSIGHT TO TIDY. `may_access` reads migrated-and-undeclared as DENY, which is right there + because every save writes an entry for every governed database — absence means an + administrator decided. No `metrics` KEY was STORABLE before this ticket, so every migrated + record in every tenant carries none, and reading that absence as DENY would revoke Metrics + for everybody on the day this shipped with nobody having decided anything. That is the exact + failure `routes_admin.get_perms` records twice already (the `ut_*` default and the surface + default), one field further down the same entry. **Only an explicit `metrics: false` + refuses.** + + ⚠ IT DOES NOT RE-ASK ACCESS. Every caller is already behind the access wall (`module_gate`, + `may_read`, `session.require`), and folding admission in here would give a denial two + possible causes with one answer — the shape that makes a permission bug take an afternoon. + """ + if perms.is_admin(user): + return True + e = entry(user, module) + return not (isinstance(e, dict) and e.get('metrics') is False) + + +def nav_may_open(user, key, st=None): + """May this account SEE `key` — in the nav, and at the route gate? The ADMISSION question. + + ⛔⛔ WHY THIS EXISTS: THE EDITOR'S DECISION REACHED THE READ DOOR AND NOTHING ELSE. Measured + on live 2026-08-18, on a real account. An administrator ticked *Odoo products* for Naomi in + Manage user and saved; the record stored `perms.product_data.access = True` and + `may_access()` agreed. **The database never appeared.** `perms.nav_pages` and + `deps.Session.require` both ask `perms.may_open` — the LEGACY `user['modules']` array — which + still read `['sales', 'customers', 'products']`, and `products` is the key of the ARCHIVED + *SKU* module, not of `product_data`. So `nav_pages` answered `['customer_data']` and the + route would have 403'd her even by URL. Two permission systems, the editor writing one and + every DOOR reading the other ([[two-permission-systems-one-armed]]). + + ⭐ THE ASYMMETRY IS THE WHOLE DESIGN, and it is not the same rule twice: + + 1. an admin sees everything (break-glass, as everywhere else); + 2. an EXPLICIT `access: false` DENIES, on any key — an administrator unticking a box must + take the row off the nav, and before this it did not; + 3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` — creator, admin or + a `core.shares` grant — and **an `access: true` entry may NEVER widen past it**. The + editor must not become a way to hand somebody another user's private table; + 4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the + editor IS the authority: `_clean_perms` writes an entry for every governed topic on + every save, so an entry means an administrator decided; + 5. anything else — a module this editor does not govern — falls through to + `perms.may_open`, UNCHANGED. + + ⛔ LEG 5 IS LOAD-BEARING AND IT IS WHY `may_access` COULD NOT SIMPLY BE CALLED HERE. + `may_access` reads migrated-and-undeclared as DENY, which is correct for a governed topic and + catastrophic for the nav: Naomi's block declares the ten governed keys and nothing else, so + `sales` would have gone from visible to denied — an outage dressed as a permission fix. + """ + if perms.is_admin(user): + return True + e = entry(user, key) + if e is not None and not bool(e.get('access', True)): + return False + if str(key or '').startswith(_ut().KEY_PREFIX): + return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) + if e is not None: + return bool(e.get('access', True)) + return perms.may_open(user, key) + + +def assistant_entry(user, module): + """The explicit database grant an Assistant snapshot may rely on, else ``None``. + + The interactive application retains an admin break-glass path and a temporary legacy-grant + compatibility path. Neither is an answer at the Assistant's app-stored data boundary: + that reader must be able to name the migrated grant that admitted a database. In particular, + a store-outage admin identity with no ``perms`` document is not an unresolved permission that + may be widened into data access. + """ + e = entry(user, module) + if (not is_migrated(user) or not isinstance(e, dict) + or not bool(e.get('access', True)) or not may_access(user, module)): + return None + return e + + +# ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── +#: ⭐⭐ W38-T16 — THE MARKER THAT SAYS "THIS COLUMN IS GOVERNED BY A GRANT", stamped once by the +#: door that creates a shared column (`routes_tables.patch_shared_cell`) and never flipped by any +#: door afterwards. It is the `perms_v` idiom one object down, and it is here for the reason that +#: marker exists: **absence must not be read as a decision.** +#: +#: ⛔⛔ WHY A MARKER AND NOT "DOES A GRANT RECORD EXIST". Every tenant-wide column shipped before +#: this ticket carries no `field` grant record, because none was STORABLE — `shares.KINDS` had +#: three members. Reading that absence as "granted to nobody" would blank every existing shared +#: column for every account the moment this arms, which is not a permission fix, it is an outage +#: (`may_read` leg 3 carries the same argument for `ut_*` keys, in the same words). +#: +#: ⛔⛔ AND IT IS WHAT MAKES THE WALL FAIL **CLOSED**. The polarity here is the opposite of every +#: other grant check in this repo: a field share is a GRANT, so "no grant" has to mean HIDDEN or +#: the wall does nothing — but "no grant record" also describes a legacy column and a store that +#: could not be read. The marker separates the three: `granted` on the column means an explicit +#: decision was taken, so an unreadable registry hides it; no marker means legacy, so nothing +#: changes. Without it, one unreadable read of `object_shares` would publish every governed +#: column to the whole tenant, silently ([[a-guard-for-the-dangerous-case]]). +#: +#: ⚠ IT IS NOT `shared_overlay`'s `"shared": True` AND MUST NOT BE CONFUSED WITH IT. That flag is +#: written and never read (D-414) — `is_shared` is a dict-membership test — so it is not evidence +#: of anything. This one is read HERE, on every assembly, and the only writer is the create door. +FIELD_GRANT_MARK = 'granted' + + +def granted_field_keys(fields): + """Every column in `fields` that declares itself GOVERNED by a `shares` grant. + + ⭐ THE CHEAP HALF OF THE WALL, AND IT IS WHY THE WALL COSTS NOTHING FOR ALMOST EVERYBODY. It + is a scan of dicts already in memory, so a database with no governed column reaches no store + at all and `hidden_keys` behaves exactly as it did before this ticket. The registry is only + opened once this answers non-empty. + """ + return {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key') + and f.get(FIELD_GRANT_MARK) is True} + + +def field_grant_hidden(user, table_key, fields, st=None): + """The governed columns of `table_key` this principal holds NO grant on — C1's per-FIELD wall. + + ⭐⭐ W38-T16 / R7 / R8 — THE THIRD WALL, AND IT COMPOSES ALONGSIDE THE OTHER TWO RATHER THAN + INSIDE THEM. `may_open` answers *IF* you reach a database; `may_read`'s stored `access: false` + overlay may REVOKE one; this answers *WHICH COLUMNS* of it you receive. It is deliberately not + threaded through that overlay: the overlay is **deny-only** by its own docstring (*"it may + revoke a database the wall below would admit, and it may never grant one that wall refuses"*) + and a field share is a GRANT — the widening direction. Merging them would give the codebase a + second idea of who grants what, which is the failure `may_open`'s own note is the record of. + It composes the way `may_open`'s grant leg does: additively, last, fail-closed. + + ⛔ AN ADMIN IS NOT WALLED (break-glass, as everywhere else) and neither is the column's OWNER — + `shares.role_for` answers `'owner'` for the creator, so a person cannot lose their own column + by forgetting to share it with themselves. + + ⚠ `st` IS THE TENANT HANDLE AND ITS ABSENCE IS SAFE HERE, unlike everywhere else. A caller + that cannot lend one reads the module-default bucket; on any tenant but #0 that finds no + grant, and no grant on a MARKED column means HIDDEN. So a door that has not learned to thread + `st` under-shares rather than over-shares, and the symptom is a grantee who cannot see their + column — visible, reportable, and the opposite of a leak. + """ + if perms.is_admin(user): + return set() + marked = granted_field_keys(fields) + if not marked: + return set() + uname = str((user or {}).get('username') or '').strip().lower() + try: + import core.shares as shares + except Exception: # noqa: BLE001 + return set(marked) + if not uname: + # No principal, and a marked column is an explicit decision: nobody is not somebody. + return set(marked) + # ⭐ THE CREATOR IS READ OFF THE COLUMN, NOT OUT OF THE REGISTRY, AND THAT IS NOT A SECOND + # AUTHORITY. `createdBy` is ALREADY what decides who may DELETE a shared column (R8 / D-172, + # `routes_tables.delete_shared_field`) and who may CLAIM it (`routes_shares._owns_object`); + # asking the same field here keeps one answer to "whose column is this" across all three. + # ⛔ IT IS ALSO THE ONLY THING THAT SURVIVES A CLAIM THAT NEVER LANDED. The create door writes + # the definition first and the grant record second, on purpose — so the window where a column + # is marked and unclaimed exists, and without this line its own author would be walled out of + # the column they just made, permanently and with no way to fix it but an admin. + mine = {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key') and str(f['key']) in marked + and str(f.get('createdBy') or '').strip().lower() == uname} + hide = set() + for key in marked - mine: + try: + oid = shares.field_oid(table_key, key) + role = shares.role_for('field', oid, uname, is_admin=False, st=st) + except Exception: # noqa: BLE001 + role = None + if role is None: + hide.add(key) + return hide + + +def _measure_bound_keys(hidden, fields): + """Every column BOUND to a measure whose `measure_` pseudo-field is in `hidden`. + + ⭐ THE PREFIX IS IMPORTED, NEVER SPELLED. `aios_grid.MEASURE_FIELD_PREFIX` is the one + constant the client's `createField`, the host's `clean_measure_field` and now this wall all + key off; a literal "measure_" here would be the third copy, and the first to drift + [[constant-two-features-share]]. The import is lazy and function-local, which is the + established shape in this layer (`core/grid_events.py`, `core/user_tables.py` both do it) and + keeps `core` from pulling the grid module at import time. + + ⚠ CHEAP FIRST. Called only once `hidden` is already non-empty, and it returns on an empty + `wanted` before touching `fields` — so a wall with no metric tick costs one set + comprehension over a handful of strings, on a function that runs at every grid door. + + ⛔ A FAILED IMPORT HIDES NOTHING EXTRA RATHER THAN TAKING THE DOOR DOWN, matching + `field_grant_hidden`'s treatment of an unreachable `core.shares`. The direction is stated + because it is the unsafe one: this leg only ever WIDENS the hidden set, so losing it + under-hides — visible, reportable, and the symptom is a metric column that should have + been walled, not a database that will not open. + """ + try: + import aios_grid as _agm + prefix = _agm.MEASURE_FIELD_PREFIX + except Exception: # noqa: BLE001 + return set() + wanted = {h[len(prefix):] for h in hidden + if isinstance(h, str) and h.startswith(prefix) and len(h) > len(prefix)} + if not wanted: + return set() + out = set() + for f in fields or (): + if not isinstance(f, dict) or not f.get('key'): + continue + spec = f.get('measure') + if isinstance(spec, dict) and str(spec.get('key') or '') in wanted: + out.add(str(f['key'])) + return out + + +def hidden_keys(user, module, fields, st=None): + """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5). + + ⛔ WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER + (`formulaEngine.ts`, injected by `computedRows`) from `{ref}`s into other columns, and a + measure column's value arrives precomputed in `derived`. So hiding field X has exactly three + possible outcomes and only one of them is coherent: + + strip X, keep formulas → every formula over X computes blank or wrong, silently + keep X's value for them → X has leaked, wearing a formula's name + strip X AND its dependents→ the only honest answer + + So a hidden field drags every formula that references it — and every formula that references + THAT formula, hence the fixpoint loop — out of the payload with it. + + ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a + graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate + anyway; without it a self-referential pair would spin here. + + ⭐⭐ W36-T21 — AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this + closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup + names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that + link points at — so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices + this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns + what the hidden link contains, in aggregate; the three outcomes are exactly the three the + formula argument above enumerates, and only "strip both" is coherent. Verified against the + real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there + is either `{'link': , 'field': }` + or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes + and `rollup.field` is deliberately not treated as one — it names another database's column, + which has its own wall. + """ + if perms.is_admin(user): + return frozenset() + # ⭐⭐ W38-T16 — TWO SOURCES OF HIDING, ONE CLOSURE, AND THAT UNION IS THE WHOLE INTEGRATION. + # + # The administrator's `hiddenFields` and a field's own grant answer different questions and + # both end in the same place: a key this reader may not receive. Seeded together HERE, before + # the fixpoint, so the transitive argument above covers the new source unchanged — a formula + # (or a rollup) over a column this reader was not granted comes out with it, or the value + # leaks wearing the derived column's name. + # + # ⛔ AND THIS FUNCTION IS THE INSERTION POINT RATHER THAN `_scoped`, WHICH IS WHAT THE TICKET + # ASSUMED. `_scoped` is C1's evaluator and reaches C1's two doors; **the product reads through + # neither of them.** Every grid door calls THIS: `routes_customers:196`, `routes_products:345`, + # `routes_odoo_tables:920/1119`, `routes_tables._ut_hidden/_ut_field_wall`, `routes_slack:190`, + # `routes_grid:57/69/787` — and `grid_events` walls WRITES off the same set through + # `EventCtx.hidden_keys`. One evaluator, every door, both wires, read and write. + hidden = set(field_grant_hidden(user, module, fields, st=st)) + e = entry(user, module) + if e: + hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k} + if not hidden: + return frozenset() + # ⭐⭐ W40-T04 / I13 — A THIRD SOURCE, SEEDED BEFORE THE FIXPOINT FOR THE SAME REASON THE + # OTHER TWO ARE, AND IT IS WHAT MAKES THE NEW CHECKBOX ENFORCE ANYTHING. + # + # The permission editor now offers one `measure_`-namespaced PSEUDO-field per bound measure + # (`routes_admin._metric_fields`), so an administrator can tick "Metric - Gross margin $" and + # the wall stores `hiddenFields: ['measure_margin']`. But the COLUMNS that carry that number + # are keyed `measure__` by `CustomerGrid.createField` — `measure_margin_a1b2`, + # not `measure_margin` — and the seeding above compares KEYS. Without this line the box + # ticks, the record saves, every gate stays green and the reader keeps every gross-margin + # column on the grid. I13's own words are *"so a user can check the ones the permissioning is + # LIMITED TO"*; a control that limits nothing is the failure + # `verify_ui.py::metrics_toggle_has_a_mount_site` exists because of. + # + # ⭐ SO THE JOIN IS ON THE BINDING, NOT THE KEY: a hidden `measure_` hides every column + # whose `measure.key` IS ``, which is the same spec `aios_grid.clean_measure_field` + # wrote and `measure_fields_of` reads. One measure, however many windows a user minted over + # it, all walled by one tick. + # + # ⭐ ADDITIVE, NEVER SUBSTITUTIVE. The pseudo-key stays in `hidden` on its own account, so a + # real column that happens to be keyed exactly `measure_margin` is hidden exactly as before + # and nothing depends on the pseudo-field existing. + # + # ⛔ AND IT GOES HERE, ABOVE THE FIXPOINT, so the transitive argument this docstring already + # makes covers it unchanged: a FORMULA over a walled metric column, or a ROLLUP whose link is + # one, comes out with it. Seeded after the loop it would leak through the derived column, + # which is outcome two of the three the docstring enumerates. + hidden |= _measure_bound_keys(hidden, fields) + + refs = {} + for f in fields or (): + if not isinstance(f, dict) or not f.get('key'): + continue + expr = f.get('formula') + if isinstance(expr, str) and expr: + refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()} + link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None + if isinstance(link, str) and link.strip(): + refs.setdefault(f['key'], set()).add(link.strip()) + + MAX_PASSES = 12 + for _ in range(MAX_PASSES): + grew = False + for key, deps in refs.items(): + if key not in hidden and deps & hidden: + hidden.add(key) + grew = True + if not grew: + break + return frozenset(hidden) + + +def visible_fields(fields, user, module, st=None): + """`fields` minus the hidden closure. Order preserved — the column order is the user's. + + ⚠ W38-T16 — `st` MUST MATCH WHAT ITS CALLER PASSED TO `hidden_keys`, and that is not a style + note. This recomputes the closure, and since the field-grant leg under-hides without a tenant + handle, a caller that lends one to `hidden_keys` and not to this would narrow the FIELD LIST + by MORE than it stripped from the ROWS — the value left sitting in the payload under a column + nobody can see, which is exactly the half-wall `strip_row`'s note exists to forbid. + """ + hide = hidden_keys(user, module, fields, st=st) + if not hide: + return list(fields or ()) + return [f for f in (fields or ()) + if not (isinstance(f, dict) and f.get('key') in hide)] + + +def assistant_visible_fields(fields, user, module): + """Visible closure under one explicit Assistant grant, including for an admin caller.""" + e = assistant_entry(user, module) + if e is None: + return [] + # `hidden_keys` deliberately preserves the interactive admin break-glass behaviour. The + # Assistant uses its explicit grant instead, but shares the same transitive formula closure. + hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k} + if not hidden: + return list(fields or ()) + # ⛔⛔ THE SAME LEAK ONE DOOR OVER, AND IT IS THE SAME `hiddenFields`. `assistant_entry` + # returns the very dict `entry()` does, so once the permission editor can store + # `measure_margin` the Assistant's grant carries it too — and comparing KEYS would leave + # `measure_gross_profit_a1b2` in the snapshot this reader is handed. Seeded here for the + # identical reason and at the identical point as in `hidden_keys`: before the fixpoint, so a + # formula over a walled metric column comes out with it. + hidden |= _measure_bound_keys(hidden, fields) + + refs = {} + for field in fields or (): + if not isinstance(field, dict) or not field.get('key'): + continue + expr = field.get('formula') + if isinstance(expr, str) and expr: + refs[field['key']] = {match.strip() for match in _FORMULA_REF.findall(expr) + if match.strip()} + for _ in range(12): + grew = False + for key, deps in refs.items(): + if key not in hidden and deps & hidden: + hidden.add(key) + grew = True + if not grew: + break + return [field for field in (fields or ()) + if not (isinstance(field, dict) and field.get('key') in hidden)] + + +def strip_row(row, hide): + """Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run + per row: the field list and the row payload are two different wires, and stripping only the + first would leave the value sitting in the second where anyone can read it.""" + if not hide or not isinstance(row, dict): + return row + return {k: v for k, v in row.items() if k not in hide} + + +def visible_overlays(overlays, user, module, fields, st=None): + """The overlay CELL MAP minus every column this reader may not receive — D-427. + + ⛔⛔ THE THIRD WIRE, AND IT CARRIES THE RAW VALUE. `strip_row`'s own docstring makes the + argument for two wires: *"the field list and the row payload are two different wires, and + stripping only the first would leave the value sitting in the second where anyone can read + it."* There is a THIRD. `/workspace?scope=product` serves + `workspace["overlays"] = g["ws"].get("overlays")` verbatim — the persisted user/tenant + stratum, keyed `{row id: {field key: value}}` — and that assignment never asks who is + reading. So an administrator hides `first_cost`, the grid dutifully drops the column and the + cell, and the same number is still sitting in the workspace payload under the same key. The + wall holds on two wires out of three, which is not a wall. + + ⭐ WHY THIS LIVES HERE RATHER THAN AT THE DOORS. `hidden_keys` is already the ONE evaluator + every grid door calls, and the three leaking assignments are one line each in three files. + Putting the narrowing beside `strip_row` means the fix at each door is a single call to the + module that already owns the question, instead of a fourth place that decides what a reader + may see. A second idea of who hides what is the failure `may_open`'s own note records. + + ⭐ SAME ARGUMENT ORDER AS `visible_fields`, deliberately: a door that already narrows its + field list has the four values to hand, and `st` MUST be the same handle it passed there. + The field-grant leg under-hides without a tenant handle, so a door that lends one to + `visible_fields` and not to this would strip the COLUMN while leaving the overlay VALUE — + the half-wall this function exists to close, re-created by the fix for it. + + ⚠ NON-DESTRUCTIVE. A new dict is built rather than the caller's mutated, because the same + overlay object is read again by `rows_from_pool` on the assembly path; and the input is + returned untouched when nothing is hidden, so a database with no wall pays one set test. + """ + hide = hidden_keys(user, module, fields, st=st) + if not hide or not isinstance(overlays, dict): + return overlays + return {rid: strip_row(cells, hide) if isinstance(cells, dict) else cells + for rid, cells in overlays.items()} + + +# ── ROWS ───────────────────────────────────────────────────────────────────────────────────── +def _wall_leaf_keys(nodes): + """Every `colId` a filter tree names, at any depth. Groups carry `children`; leaves do not. + + ⚠ THE SAME WALK AS `routes_admin._leaf_col_ids`, and the duplication is deliberate rather + than lazy: that one runs at the ADMIN DOOR inside the API package, this one runs in `core` + on the request path, and `core` never imports up (see `platform/ARCHITECTURE.md`). The two + are three lines each and `verify_api` asserts they answer the same set on the same tree, so + a grammar change cannot land in one and not the other. That equality is the point: the door + refuses through one walk and the wall resolves through the other, and a wall the editor + accepts but the enforcement path cannot see is the defect this whole change is about. + + ⛔ AN `rhs` COLUMN IS NOT COLLECTED, MATCHING `_leaf_col_ids` AND FAILING CLOSED. A + column-to-column leaf whose RIGHT side is a user-generated column stays unenriched, so + `is_rule_active` finds that side outside the contract, calls the rule inactive, and strict + mode turns that into a DENY. That is the safe answer, and it is the same one this door gave + before the enrichment existed. `prune_filter_to_fields` does walk `rhs`, deliberately, and + the asymmetry is the direction each function fails in: deleting a leaf WIDENS a wall, so + that one must see every column a leaf names; refusing to enrich one only narrows. + """ + found = set() + for node in nodes or (): + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + found |= _wall_leaf_keys(node['children']) + elif node.get('colId'): + found.add(str(node['colId'])) + return found + + +def _enrich_for_wall(tree, rows, fields, module, st): + """`(rows, fields)` — the same wall inputs, plus any USER-GENERATED column this wall names + that the SERVER can actually answer. Owner I16, the half `ut_*` got for free. + + ⭐⭐ THE DEFECT. On the two Odoo topic grids the wall runs as + `apply_row_scope(rows, user, MODULE, )` over PRE-OVERLAY rows, so a + column a user created is neither declared in that field list nor present on the row — and + `filter_eval.permits` DENIES every leaf it cannot answer. Measured four ways on the + integrated head, one leaf `{colId: , op: eq, value: West}`, two rows: + + static contract + pre-overlay rows (THE LIVE CALL SITE) -> [] every row denied + column declared + rows carrying the value -> ['1'] correct + column declared + pre-overlay rows -> [] + a DECLARED odoo column, same shape -> ['1'] the evaluator is fine + + So the administrator saves a rule, is told it saved, and that account opens an empty grid + with nothing on screen saying why. BOTH halves are needed and neither alone does anything, + which is why this function supplies both from one read. + + ⛔⛔ IT MAY ONLY EVER ADD THE ABILITY TO ANSWER — IT MUST NEVER ADMIT A ROW THE WALL WOULD + OTHERWISE REFUSE. That is why `wallable_overlay_keys` is defined as *the keys whose values + the very `cells` read below serves*, and not as "every overlay column". Merge a blank for a + key this store cannot really answer and `custom_x is not West` flips from denying every row + (undeclared leaf) to admitting every row (`'' != 'West'`), with `is empty` doing the same — + a silent widening wearing the shape of a fix. The set and the values come from ONE stratum + so they cannot disagree about what is answerable. + + ⛔ AND THE WHOLE THING IS WRAPPED FAIL-CLOSED. On ANY failure the caller's own `rows` and + `fields` come back untouched and `permits` denies exactly as it does today. A wall that + cannot be enriched is a wall that keeps refusing, never one that opens. + + ⚠ THE ORDER IS A COST DECISION, NOT A STYLE ONE. `missing` is answered from the tree and the + field list already in memory, and an empty `missing` returns BEFORE `wallable_overlay_keys` + is called — so a wall naming only declared columns (every wall that exists today) pays one + set difference and reaches no store at all. + """ + try: + from harness import filter_eval as fe + # ⛔ `tree_parts`, NEVER `tree['nodes']`. A `FilterTree` is a PAIR and a BARE LIST is also + # a legal shape (C-PERM amendment 2); a second reader of it here would answer "no leaves" + # for a wall the evaluator one line down reads perfectly well. + nodes, _conj = fe.tree_parts(tree) + leaves = _wall_leaf_keys(nodes) + if not leaves: + return rows, fields, frozenset() + declared = {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key')} + missing = leaves - declared + if not missing: + return rows, fields, frozenset() # nothing to add — the common case, and it is free + resolvable = missing + if not resolvable: # unreachable as written: `missing` is non-empty three lines up. + # Kept as the SHAPE of the guard, because `resolvable` is narrowed AGAIN below once + # the snapshot says what is actually answerable, and that narrowing can empty it. + # A wall naming a column NOTHING here can answer still denies, which is the whole + # point of `permits`. Enrichment is not a licence to ignore an unanswerable leaf. + return rows, fields, frozenset() + import core.shared_overlay as so + import core.view_templates as vt + ws_key = vt.workspace_key(str(module or '')) + if not ws_key: + return rows, fields, frozenset() + source = list(rows or ()) + # ⚠ THE ROW'S CELL KEY IS DERIVED BY `shared_overlay`'s OWN RULE, `str(int(pid))`, and + # not by `str(pid)`. That is the one coercion the stratum has (pids are ints in the grid + # and strings in JSON), so a second spelling here would look up `'1.0'` in a document + # keyed `'1'` and merge a blank over a value that is right there. It also RAISES on a + # pid-less or non-numeric row: a `cells` call carrying one would fail the whole request + # CLOSED, i.e. disable the fix silently for every other row rather than going red, so + # such a row is simply given the blank instead. + keys = {} + for i, r in enumerate(source): + if not isinstance(r, dict): + continue + try: + keys[i] = str(int(r.get('pid'))) + except (TypeError, ValueError): + continue + # ⭐⭐ ONE READ FOR THE WHOLE PAGE, AND FOR BOTH HALVES OF THE QUESTION. `snapshot` + # returns the shared stratum's SCHEMA and its CELLS from a single `st.get`, and it + # refuses an "everything" read by signature, so the row set handed in IS the bound. + # + # ⛔⛔ THE TWO HALVES MUST COME FROM ONE SNAPSHOT, AND THIS USED TO BE TWO READS. A + # wave-40 adversarial probe drove the gap: with the second read returning an emptied + # `cells`, a blank is merged for a key the store cannot really serve, and + # `custom_x is not West` flips from denying every row to ADMITTING a row whose true + # value IS West. `is empty` does the same. Narrow under today's cache-first store and + # structurally real under a threaded server, so it is closed by construction rather + # than by being unlikely. + _defs, cells = so.snapshot(ws_key, list(keys.values()), st=st) + wallable = wallable_overlay_keys(module, st=st, defs=_defs) + resolvable = resolvable & set(wallable) + if not resolvable: + return rows, fields, frozenset() + merged, denied = [], [] + for i, r in enumerate(source): + if not isinstance(r, dict): + merged.append(r) + continue + if i not in keys: + # ⛔⛔ A ROW WHOSE `pid` DOES NOT RESOLVE GETS NOTHING MERGED, AND THAT IS THE + # WHOLE POINT. `keys` holds only indices whose pid survived `str(int(pid))`; for + # any other row the store was never asked, so its value is UNKNOWN -- a different + # thing from the legitimately blank cell a real pid with no stored value has. + # Merging `''` for it hands `is not X` and `is empty` a FABRICATION and treats it + # as ground truth: a wave-40 adversarial probe drove exactly that, admitting a + # no-pid row under `is not West`, and a non-numeric-pid row under `less than 5` + # on a numeric column -- both of which the un-enriched wall refuses. Leaving the + # key off keeps the leaf unanswerable, so `permits` denies. + # + # ⚠ Not reachable through either registered reader today: `customer_data` and + # `product_data` both emit integer Odoo ids. Fixed because the invariant this + # function states is UNCONDITIONAL, not because it happened to be reachable. + # ⛔ DENIED, not merely un-merged. Leaving the key off the row is NOT + # enough: `wall_fields` DECLARES the column, and `permits` reads a declared + # field whose key is absent from the row as BLANK -- the same fabrication one + # level down, and measured doing exactly that. The index is recorded and the + # door drops the row outright. + merged.append(r) + denied.append(i) + continue + row_cells = cells.get(keys[i]) or {} + # A COPY, never the caller's dict mutated. These rows are the pool the tenant + # runtime caches and hands to every other consumer on the request (the workspace, + # the cohorts, the measures); writing a wall's working value into them would put a + # column on a payload nobody asked for it on. + merged.append(dict(r, **{k: row_cells.get(k, '') for k in resolvable})) + return (merged, list(fields or ()) + [wallable[k] for k in resolvable], + frozenset(denied)) + except Exception: # noqa: BLE001 + return rows, fields, frozenset() + + +def apply_row_scope(rows, user, module, fields, ctx=None, st=None): + """The rows this account may receive: `filter_eval.permits` over the permanent filter. + + `permits`, never `matches` — an unanswerable permanent filter DENIES rather than being + ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes + the difference a leak rather than a preference. + + ⭐⭐ `st` IS OWNER I16's SECOND HALF AND IT IS OPTIONAL SO THE FIRST HALF CANNOT MOVE. + *"Permission Filters must be able to filter on user-generated Fields too."* With a tenant + handle this door can resolve a user-generated column the static contract does not declare + (see `_enrich_for_wall`); WITHOUT one it is byte-identical to the wall that shipped before + this parameter existed, which is what keeps a cold process — a gate, a worker, E's sandbox — + behaving exactly as it always has. + + ⛔ PASS IT AT EVERY DOOR THAT WALLS A TOPIC GRID, OR AT NONE OF THEM. One stored wall read + through a door that lends the handle and a door that does not is one rule with two meanings, + which is worse than a uniform refusal: `allowed_pids` is the WRITE wall and `grid_assembly` + the READ one, and a user who may PATCH a row they cannot SEE is the hole + `allowed_pids`' own docstring exists to close. + """ + if perms.is_admin(user): + return list(rows or ()) + e = entry(user, module) + tree = (e or {}).get('filter') + if not tree: + return list(rows or ()) + from harness import filter_eval as fe + if st is not None: + # ⛔⛔ THE ENRICHMENT ANSWERS THE WALL AND MUST NEVER REACH A CALLER. It merges a + # column's value onto a COPY of each row so `permits` can evaluate a leaf naming it; + # returning those copies hands every consumer a column the caller's own field list + # does not declare. Measured in wave-40 QA against `core/script_sandbox.py`, which + # passes `scoped_table()`'s rows straight to a user-authored script: + # + # scoped_fields() declared keys: ['dba'] + # scoped_table() returned rows : [{'pid': 2, 'dba': 'Fisch', + # 'custom_region_qa': 'TOP-SECRET-VALUE'}] + # + # reachable by any ordinary session through `POST /script-views/{id}/run`. And the + # field-grant hide could never have caught it: `field_grant_hidden` can only mark a + # key already present in the `fields` it is handed, and this key never is. + # + # ⭐ So the decision is made on the enriched copy and the ORIGINAL row is what + # survives. `_enrich_for_wall` returns one entry per input row, in order, which is + # what makes the pairing sound; the copy is ONLY ever an argument to `permits`. + # ⚠ A length disagreement means the enrichment did not do what it promises, so the + # fall-through is the UN-enriched wall, which denies. Never the enriched rows. + # ⛔ MATERIALISED BEFORE THE ENRICHMENT SEES IT. `rows` may be any iterable, and + # `_enrich_for_wall` has three early-return paths that hand it straight back -- so a + # GENERATOR reached `len(judged)` and raised `TypeError: object of type 'generator' has + # no len()` on the common wall (one naming only declared columns), or, when enrichment + # did run, left the outer generator exhausted and the length check comparing against + # zero. Both fail closed, but one is a crash and the other a silent empty answer. One + # `list()` removes the class. Found by a wave-40 adversarial probe. + source = list(rows or ()) + judged, wall_fields, denied = _enrich_for_wall(tree, source, fields, module, st) + if len(judged) == len(source): + return [orig for i, (orig, seen) in enumerate(zip(source, judged)) + if i not in denied and fe.permits(tree, seen, wall_fields, ctx)] + return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)] + + +def assistant_apply_row_scope(rows, user, module, fields, ctx=None, st=None): + """Apply an explicit Assistant grant's permanent filter without an admin bypass. + + `st` carries the same meaning it does on `apply_row_scope`, for the same reason: an Assistant + grant is stored by the same editor, against the same vocabulary, and a wall that means one + thing on the grid and another in the Analyst's answer is two walls. + """ + e = assistant_entry(user, module) + if e is None: + return [] + tree = e.get('filter') + if not tree: + return list(rows or ()) + from harness import filter_eval as fe + if st is not None: + # Same rule as `apply_row_scope`: judge on the copy, return the ORIGINAL. See the block + # there for the measured leak this prevents. + # ⛔ MATERIALISED BEFORE THE ENRICHMENT SEES IT. `rows` may be any iterable, and + # `_enrich_for_wall` has three early-return paths that hand it straight back -- so a + # GENERATOR reached `len(judged)` and raised `TypeError: object of type 'generator' has + # no len()` on the common wall (one naming only declared columns), or, when enrichment + # did run, left the outer generator exhausted and the length check comparing against + # zero. Both fail closed, but one is a crash and the other a silent empty answer. One + # `list()` removes the class. Found by a wave-40 adversarial probe. + source = list(rows or ()) + judged, wall_fields, denied = _enrich_for_wall(tree, source, fields, module, st) + if len(judged) == len(source): + return [orig for i, (orig, seen) in enumerate(zip(source, judged)) + if i not in denied and fe.permits(tree, seen, wall_fields, ctx)] + return [row for row in (rows or ()) if fe.permits(tree, row, fields, ctx)] + + +def validate_assistant_filter(tree, fields): + """Return a strict, detached Assistant filter tree or raise ``ValueError``. + + The display filter cleaner is intentionally permissive: it drops a stale column or leaves an + inactive condition alone so an old saved view can still open. An Assistant data reader cannot + inherit that behaviour — dropping a predicate turns a request for a subset into a wider read. + This validator therefore admits only visible field operands that the existing evaluator can + answer from one stored row. Cohort, measure and rank conditions need separate materialised + set resolvers and are refused here rather than guessed. + """ + if tree is None: + return None + from harness import filter_eval as fe + + by_key = {str(field.get('key')): field for field in (fields or ()) + if isinstance(field, dict) and field.get('key')} + if not by_key: + raise ValueError('assistant filter has no visible field contract') + + def _leaf(raw): + if not isinstance(raw, dict): + raise ValueError('assistant filter leaf must be an object') + allowed = {'id', 'colId', 'op', 'value', 'value2', 'rhs'} + if set(raw) - allowed: + raise ValueError('assistant filter carries an unsupported operand') + col = raw.get('colId') + op = raw.get('op') + if not isinstance(col, str) or col not in by_key: + raise ValueError('assistant filter names an unknown or hidden field') + if (not isinstance(op, str) or op not in fe.FILTER_OPS or op in fe.RANK_OPS + or op in {'between', 'within'} or col == fe.COHORT_FIELD): + raise ValueError('assistant filter uses an unsupported operator') + rhs = raw.get('rhs') + if rhs is not None: + if (not isinstance(rhs, dict) or rhs.get('kind') != 'field' + or set(rhs) - {'kind', 'colId'} + or not isinstance(rhs.get('colId'), str) + or rhs['colId'] not in by_key): + raise ValueError('assistant filter names an unknown or hidden right-hand field') + out = {name: raw[name] for name in ('id', 'colId', 'op', 'value', 'value2') + if name in raw} + if rhs is not None: + out['rhs'] = {'kind': rhs.get('kind'), 'colId': rhs['colId']} + if not fe.is_rule_active(out, by_key): + raise ValueError('assistant filter is inactive or unanswerable') + return out + + def _node(raw): + if not isinstance(raw, dict): + raise ValueError('assistant filter node must be an object') + if 'children' not in raw: + return _leaf(raw) + if set(raw) - {'conj', 'children'}: + raise ValueError('assistant filter group carries an unsupported operand') + children = raw.get('children') + if raw.get('conj') not in ('and', 'or') or not isinstance(children, list) or not children: + raise ValueError('assistant filter group must be a non-empty and/or group') + return {'conj': raw['conj'], 'children': [_node(child) for child in children]} + + if isinstance(tree, list): + if not tree: + raise ValueError('assistant filter list must not be empty') + return {'conj': 'and', 'nodes': [_node(node) for node in tree]} + if not isinstance(tree, dict) or set(tree) - {'conj', 'nodes'}: + raise ValueError('assistant filters must be a tree with nodes') + nodes = tree.get('nodes') + if tree.get('conj') not in ('and', 'or') or not isinstance(nodes, list) or not nodes: + raise ValueError('assistant filter tree must be a non-empty and/or tree') + return {'conj': tree['conj'], 'nodes': [_node(node) for node in nodes]} + + +# ── USER-GENERATED COLUMNS + THE FILTER CASCADE (W40-T05 / owner I16) ──────────────────────── +def user_generated_fields(module, st=None): + """Every USER-CREATED column of `module`, across EVERY stratum -- or `None` when that + cannot be established. + + ⭐⭐ OWNER I16 — *"Permission Filters must be able to filter on user-generated Fields too. + If the field is deleted, its permission filter goes with it."* The permission editor built + its pickers from the STATIC contract (`aios_grid.FIELDS`, the product JSON), so a column a + USER made was invisible to the admin choosing what to filter on. This is the half that finds + them; `prune_filter_to_fields` below is the half that lets one go. + + ⛔⛔ `None` IS NOT `[]`, AND THE DIFFERENCE IS A SILENTLY WIDENED WALL. `[]` means + "resolved: this database has no user columns". `None` means "the vocabulary could not be + read". The cascade prunes a filter leaf naming a column that is NOT in this list, so a + DEGRADED answer would DELETE a live permission rule the moment the store was busy — + permanently, silently, and in the widening direction. Every unresolvable path therefore + answers `None` and every caller declines to prune on it. This is the GET/PUT skew + `routes_admin._clean_perms`' metric-tick note records one door over, except that there the + failure mode was a REFUSAL and here it would be a REVOCATION. + + ⚠ THE UNION OF EVERY STRATUM, DELIBERATELY. `_table_workspace` is + `{username: {views, fields, overlays}, '__shared__': {...}}` and a column lives in exactly + one of them. `_module_fields`' own docstring says this list is *"the admin choosing what to + hide"* and the SUBJECT USER IS NOT A PARAMETER OF IT, so a per-user read would make a column + unpickable for the very user who owns it. + + ⛔⛔ AND THE TENANT-WIDE STRATUM MOVED OUT OF THAT DOCUMENT, WHICH IS HOW "EVERY STRATUM" + STOPPED BEING TRUE. `core/shared_overlay.py`'s own RESIDENCY note says it: a SHARED column + lives in `_table_workspace__shared`, *"its own bucket, beside the per-user one — never + a `__shared__` member"*, and `field_permissions.promote_field` POPS the definition out of + the creator's stratum once it is promoted. So the loop below, reading one document, saw + exactly the columns nobody had shared. Measured on the live tenant: 21 of the 22 custom + columns on these two grids carry `source: "overlay", shared: true` — i.e. the function + returned the ONE column it was least useful for. + + ⛔ THE CONSEQUENCE WAS NOT A MISSING PICKER ROW, IT WAS A SILENT REVOCATION. This list is + also `routes_admin._prunable_vocabulary`'s answer, and `prune_filter_to_fields` DELETES a + leaf naming a column outside it. A wall stored against a shared column would therefore have + had its leaf pruned on the next read of the record — permanently, silently, and in the + widening direction, which is the exact failure the `None`-is-not-`[]` note above exists to + prevent, arriving through the door this function opens. + + ⚠ A SHARED READ THAT RAISES ANSWERS `None`, LIKE THE PER-USER ONE; an ABSENT shared bucket + is "nothing has been shared here yet" and does NOT poison the answer. Both arms are + deliberate: the first keeps the vocabulary honest under contention, and the second is what + stops a tenant that has never shared a column from losing the per-user half of I16. + + ⛔ THE BUCKET NAME IS `view_templates.workspace_key`, NEVER SPELLED. `customer_data`'s bucket + is `customer_table_workspace` — the module key is NOT the storage key — and `_WS_KEYS` is + already the one place that mapping lives ([[one-question-two-normalizers]]). + + ⛔ THE SHAPE COMES FROM `aios_grid.fields_from_workspace`, NEVER HAND-BUILT. That is the + function the GRID overlays a saved stratum with, so a column reaches the permission editor + described exactly as the user sees it — including `filterable: False` on a `measure_` column, + whose value is host-computed into `derived` and never sits on a row. A hand-shaped dict here + would be a second idea of what a field is, and the first thing it would lose is that flag. + `fields_base=[]` makes the base loop a no-op, so what comes back is the SAVED stratum alone. + + ⚠ `scope_key=None` DROPS A COHORT-SCOPED COLUMN, AND THAT IS THE ANSWER RATHER THAN A GAP. + `grid_events` writes `scope: 'cohort'` on a column created from the Cohort surface, and such + a column is not on the Customer grid's rows at all. Offering it to the permanent filter would + admit a leaf that can only ever DENY every row — the exact reason `_metric_fields` marks a + metric pseudo-field unfilterable. The narrow read is the honest one here. + """ + if st is None: + return None + try: + import aios_grid + import core.shared_overlay as shared_overlay + import core.view_templates as view_templates + except Exception: # noqa: BLE001 + return None + bucket = view_templates.workspace_key(str(module or '')) + if not bucket: + return None # a SURFACE, or nothing this layer knows as a table + try: + doc = st.get(bucket) + except Exception: # noqa: BLE001 + return None + if not isinstance(doc, dict): + # ⛔ AN ABSENT BUCKET IS `None`, NOT `[]`. A store that cannot answer and a database + # nobody has opened are indistinguishable from here, and only one of them is safe to + # prune against. Declining costs nothing real: a workspace with no bucket has no column + # to have deleted either. + return None + # ⛔ READ THROUGH `st` RATHER THAN `shared_overlay.fields()`, AND ONLY THE NAME COMES FROM + # THAT MODULE. `shared_overlay._read` swallows every exception and answers `{}`, so an + # unreadable store would be indistinguishable from "nothing is shared" — which is precisely + # the `None`-collapsed-into-`[]` this function refuses everywhere else. `bucket()` is still + # the ONE spelling of the key, so there is no second naming convention to get wrong. + try: + shared_doc = st.get(shared_overlay.bucket(bucket)) + except Exception: # noqa: BLE001 + return None + strata = list(doc.values()) + if isinstance(shared_doc, dict): + strata.append(shared_doc) + out, seen = [], set() + for blob in strata: + if not isinstance(blob, dict): + continue + saved = blob.get('fields') + if not isinstance(saved, dict) or not saved: + continue + try: + got = aios_grid.fields_from_workspace({'fields': saved}, fields_base=[]) + except Exception: # noqa: BLE001 + # One malformed stratum must not make the whole vocabulary unresolvable — that would + # turn a bad saved field into a frozen cascade. Skip it; the other strata still count. + continue + for f in got or (): + key = f.get('key') if isinstance(f, dict) else None + if not key or key in seen: + continue + seen.add(key) + out.append(f) + return out + + +def wallable_overlay_keys(module, st=None, defs=None): + """`{key: Field}` — the user-generated columns of `module` a PERMANENT FILTER can be + evaluated against on the server. Owner I16, narrowed to what is true rather than to what is + offered. + + ⭐⭐ THE DEFINITION IS "WHOSE VALUES `shared_overlay.cells` SERVES", AND EVERY OTHER PROPERTY + FOLLOWS FROM IT. `_enrich_for_wall` merges these keys onto the rows from exactly one read of + exactly this bucket, so defining the set any other way would let it name a column whose value + the merge cannot supply — and a blank merged for a column the store cannot really answer + turns `is not X` and `is empty` from "denies every row" into "admits every row". The set and + the values therefore come from ONE stratum, by construction rather than by care. + + ⛔ SO A `formula`, `created_time` OR `measure_*` COLUMN CAN NEVER BE IN HERE, AND IT IS + `aios_grid.fields_from_workspace` THAT SAYS SO RATHER THAN A LIST OF TYPE NAMES. That is the + normaliser the GRID itself overlays a saved stratum with: it emits the read-only user pair + and the measure columns as `source: 'odoo', derived: True` (their values are computed in the + BROWSER, or from the measure catalogue, and never sit on a stored row) and an editable + overlay column as `source: 'overlay'` with no `derived` at all. Testing the two flags is + testing the grid's own declaration; a hand-built type list here would be a second idea of + what a field is, and the first thing it would lose is the next read-only kind somebody adds. + + ⛔ AND A PER-USER PRIVATE OVERLAY COLUMN IS OUT TOO, which is the part that looks like a gap + and is not. Its values ARE server-readable (`[username]['overlays']`), but they are + readable only for the SUBJECT of the wall — an account that may edit that column freely, and + would therefore be one cell edit away from walking out of its own permission wall. The admin + who wrote the rule cannot even see the values. `routes_admin._row_wall_blind_keys` keeps + refusing those at the write door, where a person is present to choose a shared column + instead. + + ⚠ `{}` ON ANY FAILURE, NEVER `None` AND NEVER A PARTIAL SET. This is read by a wall, and the + only safe degraded answer for a wall is "I can answer nothing extra" — which leaves `permits` + denying an unresolvable leaf exactly as it does today. A half-resolved set could ADMIT a row, + which is the one direction this must never fail in. + + ⛔ AND IT ANSWERS `{}` FOR A `ut_*` DATABASE — DELIBERATELY, NOT BY ACCIDENT OF THE KEY. The + two topic grids name their shared stratum off the WORKSPACE key + (`customer_table_workspace__shared`), while a user table names its own off the TABLE key + (`routes_tables._ut_shared_fields` reads `shared_overlay.fields(table_key)`, i.e. + `ut_leads__shared`, not `ut_leads_table_workspace__shared`). Deriving from `workspace_key` + therefore finds nothing on a `ut_*` key, and that is the right answer TODAY rather than a + gap to paper over: a `ut_*` wall is validated by `routes_admin._module_fields` against the + database's own stored DEFINITION, which the enforcement path already declares and whose rows + already carry the values — so there is nothing for this to add, and the permission editor + cannot offer a `ut_*` shared column in the first place. Wiring that stratum in is a change + to what an admin may WALL ON, and it belongs with the picker change that offers it. + """ + if st is None: + return {} + try: + import aios_grid + import core.shared_overlay as shared_overlay + import core.view_templates as view_templates + + bucket = view_templates.workspace_key(str(module or '')) + if not bucket: + return {} + # ⭐ `defs` IS THE CALLER'S SNAPSHOT, AND PASSING IT IS NOT AN OPTIMISATION. + # `_enrich_for_wall` derives the VALUES from one read of this bucket and the + # answerable SET from this function; taking a second read here would let the + # two disagree, and a set that names a key the values cannot serve is exactly + # how a blank gets merged and `is not` flips from deny-all to admit-all. See + # `shared_overlay.snapshot`. + doc = ({'fields': dict(defs)} if defs is not None + else st.get(shared_overlay.bucket(bucket))) + saved = doc.get('fields') if isinstance(doc, dict) else None + if not isinstance(saved, dict) or not saved: + return {} + out = {} + for f in (aios_grid.fields_from_workspace({'fields': saved}, fields_base=[]) or ()): + if not isinstance(f, dict) or not f.get('key'): + continue + if f.get('source') != 'overlay' or f.get('derived'): + continue + out[str(f['key'])] = f + return out + except Exception: # noqa: BLE001 + return {} + + +def prune_filter_to_fields(tree, valid_keys): + """`(tree, dropped)` — `tree` with every leaf naming a column outside `valid_keys` removed. + + ⭐⭐ THE CASCADE OWNER I16 ASKS FOR: *"If the field is deleted, its permission filter goes + with it."* A stored permanent filter naming a column the database no longer has is not + ignored by the wall — `apply_row_scope` uses `permits`, which DENIES anything it cannot + answer — so a deleted column silently converts that account's grid to zero rows. Dropping + the leaf is what the owner accepted instead. + + ⛔⛔ THIS IS AN ADMIN-DOOR OPERATION AND IT MUST NEVER MOVE INTO THE ENFORCEMENT PATH. + `verify_perm_scope`'s row-scope leg asserts, in these words, *"a wall naming a DELETED column + denies every row (never ignored)"* — it puts `{'colId': 'ghost', ...}` in a stored filter and + calls `apply_row_scope` directly. If the wall started ignoring unknown leaves, that gate would + flip in the FAIL-OPEN direction and a real permission wall would quietly stop walling. So the + prune runs where a RECORD is read or written, and the wall keeps denying as the backstop for + anything the prune has not reached yet. + + ⛔ PER LEAF, WHERE `routes_admin._prune_to_module` IS ALL-OR-NOTHING PER GROUP, AND THE + DIFFERENCE IS DELIBERATE. That function folds a LEGACY wall into a module that may not be + able to evaluate it, where half a group is a rule nobody wrote. This one answers a different + question: one named column is GONE, and the owner's instruction is that its leaf goes with it + while the rest of the admin's rule stands. `done-when` says so in as many words -- *"leaving + the user's other filters intact"*. + + ⚠ AND THE WIDENING IS REAL, SO IT IS STATED RATHER THAN BURIED. Dropping a leaf from an `and` + group WIDENS that wall, and dropping the last leaf anywhere removes it altogether. That is + the direction the owner chose over deny-every-row; it is also why `valid_keys` must be a + RESOLVED vocabulary (see `user_generated_fields`) and never a degraded one. + + ⚠ A LEAF'S `rhs` COUNTS AS NAMING A COLUMN. A column-to-column comparison whose right side + was deleted is just as stale as one whose left side was, and `clean_filter_tree` would answer + it by dropping the `rhs` alone -- turning "revenue > forecast" into a comparison against a + literal, which is a DIFFERENT question wearing the original's shape. + + ⛔ AN EMPTIED TREE IS `None`, NEVER `{'conj': 'and', 'nodes': []}`. Measured: `permits` + returns True on an empty node list, so an empty-but-present tree admits every row -- while + `wall_declared` and `row_scope_applies` both read it as TRUTHY and answer that a wall applies. + A door would then build rows to filter them against nothing, and an editor would paint a rule + that is not there. `None` is the one shape all three agree about. + """ + valid = {str(k) for k in (valid_keys or ())} + if not valid: + # ⛔ A FLOOR, NOT AN OPTIMISATION. An empty vocabulary would prune EVERY leaf, which for a + # module whose schema momentarily failed to resolve is the whole wall gone in one read. + return tree, [] + dropped = [] + + def _keep(node): + if not isinstance(node, dict): + return None + kids = node.get('children') + if isinstance(kids, list): + surviving = [k for k in (_keep(k) for k in kids) if k is not None] + # An emptied GROUP carries no meaning once persisted -- the same rule + # `clean_filter_tree` applies to one it built. + return dict(node, children=surviving) if surviving else None + named = [node.get('colId')] + rhs = node.get('rhs') + if isinstance(rhs, dict) and rhs.get('kind') != 'stat' and rhs.get('colId') is not None: + named.append(rhs.get('colId')) + stale = [str(c) for c in named if c is not None and str(c) not in valid] + if stale: + dropped.extend(stale) + return None + return node + + if not isinstance(tree, dict): + return tree, [] + nodes = tree.get('nodes') + if not isinstance(nodes, list): + return tree, [] + kept = [n for n in (_keep(n) for n in nodes) if n is not None] + if not dropped: + return tree, [] # unchanged by construction, not merely equal + if not kept: + return None, sorted(set(dropped)) + return {'conj': 'or' if tree.get('conj') == 'or' else 'and', 'nodes': kept}, \ + sorted(set(dropped)) + + +# ── THE PUSHDOWN ───────────────────────────────────────────────────────────────────────────── +#: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly +#: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission +#: filter admitting Fisch admits {Fisch, Both} — a Both customer IS a Fisch customer. +_DBA_TEAM = ((frozenset({'fisch', 'both'}), 5), (frozenset({'royal', 'both'}), 6)) + +#: Only equality pushes down. `FILTER_OPS` is single-valued (there is no column-level "is any +#: of" — that vocabulary belongs to cohort leaves and is disjoint), so a multi-value BU +#: condition arrives as an OR GROUP of `eq` leaves, handled by `_group_dba_team` below. +_PUSHDOWN_OPS = frozenset({'eq'}) + +#: ⭐ THE MODULES WHOSE FIELD VOCABULARY CAN EXPRESS A BUSINESS UNIT — i.e. whose canonical +#: contract carries a `dba` column. R1's "BU access is just a permanent filter" holds only on +#: these; on every other topic there is no column to write the condition against, so a filter +#: literally cannot say it and the record's `bus` remains the only place the fact lives. +#: +#: `product_data` is the counter-example that made this constant necessary: 19 fields, no brand +#: column, and a product's BU is a property of WHOSE ORDERS built its revenue rather than of the +#: SKU. Without the fallback below, a Fisch-only account read the product catalogue with +#: Fisch+Royal money on every row — the amendment-3 defect, arriving through the door amendment 3 +#: was written to close. +#: +#: ⚠ A LIST THAT MIRRORS A JSON CONTRACT DRIFTS UNLESS SOMETHING CHECKS IT. `verify_api` asserts +#: membership here matches "this topic's contract has a `dba` field" for every governed module, so +#: a topic that grows or loses a brand column cannot silently keep the wrong rule. Named in this +#: module rather than derived from `aios_grid` on purpose: `perm_scope` is on the API's request +#: path and importing the grid stack to answer a two-element question is a cost per request. +BU_FILTERABLE_MODULES = frozenset({'customer_data'}) + + +def derive_pool_scope(user, module): + """`(team_id, agent)` the POOL must be BUILT with, derived from the permanent filter. + + ⛔ THIS EXISTS BECAUSE `team_id` SHAPES VALUES, NOT ROW MEMBERSHIP (C-PERM amendment 3). + `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, + LY, LTM) and into `cust._cadence_bulk`, so it decides what `rev`, `ly`, `ltm`, `aov`, + `est_missed` and the derived `status` MEAN. Enforce a BU purely as a post-filter and a + Fisch-only user keeps a correct-looking row LIST while every number on it silently becomes + Fisch+Royal — worst for `dba = Both` customers, who are exactly the ones a BU filter admits. + A pid-level reconciliation cannot see that; the values one can, and does. + + So the query-level pushdown SURVIVES — but as a DERIVATION OF the permanent filter rather + than a second wall beside it, which is what keeps R1's "BU access is just a filter" true at + the level the owner asked for it (one declaration, one UI, one engine). + + Pure, and recomputed per request rather than stored: a stored derivation drifts from the + filter it came from, and then two things disagree about what an account may see. + + Reads TOP-LEVEL AND-conjunction leaves ONLY. A leaf under `or` guarantees no narrowing — + `dba is Fisch OR revenue > 10` must not pin the pool to Fisch — so it never pushes down. + Anything not recognised here simply is not pushed down; `apply_row_scope` still applies the + whole tree, so the wall is unchanged either way. Belt AND braces, deliberately: the pushdown + is what makes the VALUES right, `permits()` is what makes the ROWS right. + + ⭐ THE `bus` FALLBACK, AND WHY IT IS NOT A HOLE IN AMENDMENT 4. On a topic outside + `BU_FILTERABLE_MODULES` there is no column a BU condition could be written against, so + "the filter pins no team" cannot mean "the admin chose consolidated" — it is the only answer + the filter language has. Resolving that silence as None returns the WIDER scope, which makes + the current code fail-OPEN on the values axis for exactly the topic that cannot argue back. + So the record's own `bus` answers instead, and the direction is what makes it safe: this can + only ever REPLACE None (both units) with a pinned single unit. It never widens, it never + touches `may_access`, and a `bus:'all'` account is unaffected because `scope_team_id` returns + None for it — which is every account in tenant #0's registry except the one this shipped for. + """ + if perms.is_admin(user): + return None, None + team_id, agent = _derive_from_filter(user, module) + if team_id is None and module not in BU_FILTERABLE_MODULES: + team_id = perms.scope_team_id(user) + return team_id, agent + + +def _derive_from_filter(user, module): + """`(team_id, agent)` the PERMANENT FILTER pins, before any fallback. Split out so the + fallback has exactly one place to apply — the three exits below all mean "the filter pinned + nothing", and a rule written at each of them is a rule that will one day be written at two.""" + e = entry(user, module) + tree = (e or {}).get('filter') + if not tree: + # Un-migrated records still answer through the legacy derivation, so the old wall keeps + # working until the migration has run. + if not is_migrated(user): + return perms.scope_team_id(user), perms.scope_agent(user) + return None, None + + from harness import filter_eval as fe + nodes, conj = fe.tree_parts(tree) + if conj == 'or': + return None, None + + team_id, agent = None, None + for n in nodes: + if not isinstance(n, dict): + continue + if isinstance(n.get('children'), list): + # A top-level OR GROUP under an AND root IS a guaranteed narrowing — every row must + # satisfy it — so it may push down, unlike a leaf under an OR ROOT (refused above). + # This is the shape a multi-value BU condition actually takes; see `_group_dba_team`. + tid = _group_dba_team(n) + if tid is not None: + team_id = tid if team_id in (None, tid) else None + continue + if n.get('op') not in _PUSHDOWN_OPS: + continue + col = n.get('colId') + raw = n.get('value') + if col == 'dba': + vals = {v.strip().lower() for v in str(raw or '').split(',') if v.strip()} + if not vals: + continue + for allowed, tid in _DBA_TEAM: + if vals <= allowed: + # Both BUs named = no narrowing to push; leave it to the post-filter. + team_id = tid if team_id in (None, tid) else None + break + elif col == 'agent': + v = str(raw or '').strip() + # A SET of agents cannot become the pool's single `agent_name`; the post-filter + # handles it. Only an unambiguous single value pushes down. + if v and ',' not in v: + agent = v + return team_id, agent + + +def _group_dba_team(group): + """The team a top-level `or` group pins, or None. + + Recognises ONLY the exact shape "every child is a `dba eq ` leaf" — the group the + condition builder emits for a multi-value BU condition, and the one `perm_migrate` writes. + Every OTHER group returns None and is left entirely to the post-filter: a group mixing `dba` + with another column, or containing a nested group, does not pin a BU on its own, and + guessing that it does would build the pool from the wrong book. Narrow by construction — + the pushdown may only ever be an OPTIMISATION of a constraint the filter already expresses. + """ + if group.get('conj') != 'or': + return None + children = group.get('children') or [] + if not children: + return None + vals = set() + for c in children: + if (not isinstance(c, dict) or isinstance(c.get('children'), list) + or c.get('colId') != 'dba' or c.get('op') != 'eq'): + return None + v = str(c.get('value') or '').strip().lower() + if not v: + return None + vals.add(v) + for allowed, tid in _DBA_TEAM: + if vals <= allowed: + return tid + return None + + +# ── C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ──────────────────────────────── +#: ⭐⭐ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same +#: permission logic, always"* — per-user field visibility AND row filtration on every database +#: carrying a unique id, whatever created it, with a NEW database inheriting it by construction +#: rather than by a list somebody maintains. +#: +#: ⛔ THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line +#: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic +#: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone — +#: creator, admin, or a `core.shares` grant — which is a BINARY door: you see all 31,418 rows of +#: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this +#: work in as many words (*"Arming `perm_scope` over `ut_*` … booked, not faked"*), and owner +#: item 11 is that booking coming due. +#: +#: ⚠ AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT +#: user-created. Ten of them (`ut_odoo_invoices`, `…_orders`, `…_agents`, `…_accounts`, `…_bills`, +#: `…_vendors`, `…_order_lines`, `…_gl_lines`, `…_customers`, `…_products`) are generated by the +#: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a +#: statement about origin**, and a wall keyed off it was reading a naming artefact as a security +#: boundary. +#: +#: ⛔⛔ THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and +#: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES +#: them — it calls `may_open`, it does not reimplement it — because merging them is how this +#: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20 +#: note). One resolver per question, asked in order. + + +class UnknownTable(LookupError): + """No database in this tenant answers to that key. + + ⛔ RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this + database is empty"* — indistinguishable from a real empty table, and the caller least able to + notice is the one that wanted rows. This repo has shipped that exact silent-empty answer + before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]). + """ + + +class Denied(PermissionError): + """This principal may not read this database at all. The IF question, answered by `may_read`.""" + + +class Unresolvable(RuntimeError): + """The rows exist and cannot be served under this call's constraints — R6's SECOND SENTENCE. + + ⭐ STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there + is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a + limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never + silently enforced as a short answer. Carries the same four keys + `routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight + to a client without a second vocabulary ([[one-question-two-normalizers]]). + """ + + def __init__(self, subject, effect, cause, recommendation): + self.subject, self.effect = subject, effect + self.cause, self.recommendation = cause, recommendation + super().__init__(f"{subject}: {effect}. {cause}. {recommendation}") + + def as_limit(self): + """The dict shape `routes_tables` puts in an assembly's `limits` list.""" + return {"subject": self.subject, "effect": self.effect, + "cause": self.cause, "recommendation": self.recommendation} + + +#: Row readers DECLARED by the app layer, keyed by EXACT database key. +#: `reader(table_key, user, st) -> (fields, rows)`. +#: +#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and +#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache +#: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables` +#: already uses for exactly this reason — `register_connected`, `register_read_through`, +#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being +#: interrogated by it."* +_ROW_SOURCES = {} + +#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. +#: `reader(table_key, field_keys, st) -> rows`. +_MIRROR_READER = None + + +def register_rows(reader, *table_keys): + """Declare who reads a NAMED database's rows. Returns the registered key set. + + ⚠ The return value is the registrar's own answer on purpose: a public function whose only + caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that + says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write + door's return keeps one construction site of the set instead of two. + """ + for key in table_keys: + k = str(key or '').strip() + if k: + _ROW_SOURCES[k] = reader + return frozenset(_ROW_SOURCES) + + +def register_mirror(reader): + """Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`).""" + global _MIRROR_READER + _MIRROR_READER = reader + return _MIRROR_READER is not None + + +#: ⛔ `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE +#: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE +#: list to read"* — and `register_rows` ALREADY returns exactly that, which is the same idiom +#: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through +#: the write door's return keeps ONE construction site of the set. A second accessor beside it is a +#: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py` +#: — the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by +#: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the +#: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered. + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def may_read(user, table_key, st=None): + """May this principal read `table_key` AT ALL — the IF question, on EVERY database. + + ⛔ COMPOSED, NOT RE-DERIVED, and the order is the whole rule: + + 1. an admin reads everything (break-glass — `deps._user_for` hands back a hardcoded master + dict on a store outage and it will never carry a `perms` block); + 2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner + item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall + below would admit, and it may never grant one that wall refuses; + 3. a `ut_*` database defers to `user_tables.may_open` — creator, admin, or a `core.shares` + grant — UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."* + 4. anything else is a registry topic and defers to `may_access` above. + + ⛔⛔ WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does. + `may_access` reads migrated-and-undeclared as DENY — correct for a topic, because + `routes_admin` writes an entry for every governable topic on every save. ⚠ NO `ut_*` ENTRY WAS + STORABLE AT ALL UNTIL W36-T22 — `_clean_perms` refused the key with `unenforced_module` — so + every record migrated before this wave carries no entry for any of them, and reading that + absence as a decision would revoke all ten keychain databases from every migrated account the + moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been + answering rather than the marker that has not, and it keeps being right AFTER the flag's + deletion: an admin who has never opened the editor for a database has still not decided + anything about it. + + ⛔⛔ PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a + stored record is keyed BY username in that bucket and does not carry one INSIDE it — only + `core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand + this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*` + database reads as DENIED for an account that can open all of them. It fails in the SAFE + direction and is silently wrong, which is the worst pair to debug — it cost two call sites in + one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage. + """ + if perms.is_admin(user): + return True + e = entry(user, table_key) + if e is not None and not bool(e.get('access', True)): + return False + key = str(table_key or '') + if key.startswith(_ut().KEY_PREFIX): + return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) + return may_access(user, table_key) + + +def wall_declared(user, table_key): + """Is a ROW or FIELD narrowing declared for this principal on this database? + + ⛔ THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a + stored `ut_*` wall would be INERT — *"the editor would say DENY, the table routes would keep + serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore + REFUSE for a principal this returns True for, rather than serve the whole database. False for + an admin (they bypass the wall entirely) and for any record with no entry, so a door asking + this pays nothing and changes nothing for everybody who has no wall. + """ + if perms.is_admin(user): + return False + e = entry(user, table_key) + if not e: + return False + return bool(e.get('filter')) or bool(e.get('hiddenFields')) + + +def row_scope_applies(user, table_key): + """Does a permanent ROW filter narrow this principal on this database? + + ⚠ `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows + it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its + whole point is that the pid set costs no row pass, and paying for one on every database + switch — for every account, walled or not — would undo W30-T30 to enforce a rule that applies + to almost nobody. Asked here rather than spelled out at the call site, so there is ONE + statement of when the row wall bites ([[one-question-two-normalizers]]). + """ + if perms.is_admin(user): + return False + return bool((entry(user, table_key) or {}).get('filter')) + + +def scoped_table(user, table_key, st=None, ctx=None): + """⭐⭐ CONTRACT C1 — the rows of ANY database, already field-stripped and row-filtered for + `user`. Registry topic or `ut_*`; there is no third kind and no per-database branch. + + rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database + rows = scoped_table(user, 'customer_data') # a registry topic + + `user` is a user RECORD (the dict `deps.Session.user` carries), not a username — the whole + wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller + that forgets the principal must not run, because the only thing a defaulted one could mean is + "unscoped", which is the widening direction. + + ⛔ FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the + right status instead of guessing: `UnknownTable` (no such database — never an empty list), + `Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is + why — standing rule 1's second sentence). + + ⚠ NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only + thing that stops it is a population that exceeds one materialisation window, and that arrives + as `Unresolvable` carrying its cause and a recommendation rather than as a short answer. + + ⭐ E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is + THE door rather than A door: everything a sandboxed script may read, it reads here, under the + CALLING user's scope (R5). + """ + _fields, rows = _scoped(user, table_key, st=st, ctx=ctx) + return rows + + +def scoped_fields(user, table_key, st=None): + """The COLUMNS of any database this principal may see — C1's other half. + + ⛔ IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the + field list and the row payload are two different wires, and narrowing one without the other + leaves the value sitting where anything can read it. A caller that must render a scoped table + needs both, and E cannot read a `ut_*` definition to learn its columns — the sandbox has no + second path to the store (W1). So both come from here, off one wall. + + ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a + registry topic it goes through the registered reader, which builds that topic's pool; the + pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. + """ + fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) + return fields + + +def _scoped(user, table_key, st=None, ctx=None, want_rows=True): + """`(fields, rows)` — ONE evaluator behind both public doors, so they cannot disagree.""" + key = str(table_key or '').strip() + if not key: + raise UnknownTable('a database key is required. This door will not guess which database ' + 'was meant') + if not may_read(user, key, st=st): + raise Denied(f"this account may not read '{key}'") + fields, rows = _read(key, user, st, want_rows) + # THE FIELD WALL — a TRANSITIVE closure, so hiding a column also hides every formula computed + # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference + # is the wrong shape here. + # ⭐ W38-T16 — `st` RIDES INTO THE WALL, not just into the read. The field-grant leg resolves + # against `object_shares` in THIS tenant's store; without the handle it would answer from the + # module default (tenant #0) and hide a grantee's own column on every other tenant. + hide = hidden_keys(user, key, fields, st=st) + if not want_rows: + return (visible_fields(fields, user, key, st=st) if hide else fields), [] + # THE ROW WALL — `permits()`, so a permanent filter this evaluator cannot answer DENIES + # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a + # permanent filter may name a column the reader is not allowed to SEE, and dropping the + # predicate would widen the read rather than narrow it. + # ⭐ OWNER I16 — `st` RIDES INTO THE ROW WALL TOO, and this door is the one that already had + # the handle and simply did not pass it down. The field wall two lines up has taken it since + # W38-T16 for the same reason: a wall resolved without a tenant handle answers from the + # module default, and here that means a user-generated column reads as unanswerable and + # denies every row. + rows = apply_row_scope(rows, user, key, fields, ctx, st=st) + if hide: + fields = visible_fields(fields, user, key, st=st) + rows = [strip_row(r, hide) for r in rows] + return fields, rows + + +def _read(table_key, user, st, want_rows=True): + """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" + reader = _ROW_SOURCES.get(table_key) + if reader is not None: + fields, rows = reader(table_key, user, st) + return list(fields or ()), list(rows or ()) + ut = _ut() + if not table_key.startswith(ut.KEY_PREFIX): + # ⛔ A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never + # imported the API layer this is the honest answer: nothing here can build that pool. + raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader " + f"is registered for it") + return _read_user_table(table_key, st, want_rows) + + +def _read_user_table(table_key, st, want_rows=True): + """core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately. + + ⭐ WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same + argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold + process — E's sandbox subprocess, a worker, a gate — that never imported an API route still + owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and + the sandbox is exactly such a process. + + ⚠ THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions + without the 28.6 MB of rows (D-213), which is every read this function makes when + `want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so + the materialised arm below takes the whole read explicitly. + """ + ut = _ut() + lent = ut.lend_defs(st) + defn = ut.get(table_key, st=lent) + if not defn: + raise UnknownTable(f"no database named '{table_key}'") + fields = [dict(f) for f in (defn.get('fields') or [])] + if not want_rows: + return fields, [] + if not ut.materialises(table_key, st=st, defn=defn): + # A read-through grid stores no rows here — they live in the mirror, and reading + # `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking, + # wrong, and silent. + if _MIRROR_READER is None: + raise Unresolvable( + subject='rows', effect='unreadable', + cause=(f"'{table_key}' is served read-through from the connector mirror and no " + f'mirror reader is registered in this process'), + recommendation=('call `perm_scope.register_mirror(...)` from the app layer before ' + 'reading a read-through database, or read it through the API')) + keys = {f['key'] for f in fields if f.get('key')} + return fields, list(_MIRROR_READER(table_key, keys, st) or ()) + whole = ut.get(table_key, st=st) + if whole is None: + # Deleted between the wall and here. The same refusal, not an empty table. + raise UnknownTable(f"no database named '{table_key}'") + field_keys = {f['key'] for f in fields if f.get('key')} + rows = [] + for rid, row in (whole.get('rows') or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r['pid'] = int(rid) + rows.append(r) + rows.sort(key=lambda r: r['pid']) + return fields, rows + + +# ── C10 / R12: THE PIVOT WALL — BOTH SIDES OF ONE JOIN (wave 41, W41-T16) ───────────────────── +#: ⭐⭐ A PIVOT MUST NEVER BECOME A WAY TO READ A DATABASE YOU ARE WALLED OUT OF. R10's pivot set +#: is "products ordered by the filtered customers": one request, TWO databases, and the account +#: that made it holds a SEPARATE verdict on each. Everything above this line answers for one +#: database at a time, so a caller wiring C10 by hand would naturally wall the side it was already +#: looking at and let the other one through — which is exactly the read the source-side wall was +#: installed to prevent, arriving through a door built after it. +#: +#: ⛔ AND THE FAILURE IS SILENT IN THE WORST DIRECTION. A pivot that answers a walled-out reader +#: with `rows: []` is indistinguishable from a pivot answering "this customer ordered nothing": +#: the reader draws the BUSINESS conclusion, not the permission one, and nothing anywhere says +#: otherwise. That is why C10 carries `refusal` as a first-class response key instead of letting +#: an empty grid stand in for one, and why the refusal names WHICH database refused rather than +#: answering a bare boolean ([[empty-answer-vs-unfinished-answer]]). +#: +#: ⛔ R12 — RELATIONAL SCOPE IS EXACTLY TWO DATABASES THIS WAVE: *"the permission wall applied on +#: BOTH sides of the join. ⛔ NOT chaining to a third database."* A third hop is refused here BY +#: NAME rather than left undefined, because "undefined" in a wall means whatever the first caller +#: happens to do, and the first caller is in another lane. +#: +#: ⚠ THIS IS THE SERVER HALF OF A CLIENT FEATURE THAT DOES NOT EXIST YET. `pivot_scope` is the +#: only door; a route that resolves one side itself has re-derived half a wall, which is how this +#: codebase got two ideas of who owns a table once already (`may_read`'s note above). + +#: R12's path length, stated ONCE so the refusal and the response envelope cannot disagree about +#: what "a pivot" is. +PIVOT_PATH_LEN = 2 + + +class PivotScope: + """⭐⭐ CONTRACT C10 — the verdict on ONE pivot, with BOTH sides of the join already resolved. + + Built by `pivot_scope()` and by nothing else: a hand-built one is a verdict nobody asked the + wall for. + + It holds the PRINCIPAL it was resolved for, and that is the property which makes it safe to + pass around. `source_grid`/`target_grid` cannot be handed a different user than the one the + verdict was computed against, so a route cannot resolve the scope for the caller and then + filter rows for somebody else. + + ⛔ ON A REFUSAL, `source_scope` AND `target_scope` ARE `None`, NOT `(None, None)`. The tuple + form is a legitimate answer meaning "no narrowing" — the WIDEST scope — so handing it back for + a database this account may not read at all would let a caller that ignored the refusal unpack + a pass. `None` raises on unpacking instead, which is the direction this file fails in + everywhere else. + + `code` is SERVER-SIDE ONLY and never reaches the wire; C10's refusal is `{reason, subject}` + and exactly those two keys. The closed set a route may branch on is: + `no_database` (a side was not named) · `not_relational` (that key is not a joinable database) + · `chained` (R12: a third hop) · `unreadable` (the wall said no) · `unresolvable` (the wall + could not be asked, so it refuses). A route wanting a status: `unreadable` is 403, + `no_database`/`not_relational`/`chained` are 400, `unresolvable` is 401 when `subject` is + `'session'` and 503 otherwise. ⚠ But C10's answer is a 200 carrying `refusal` — a status code + is a SECOND channel, and only one of them renders the sentence the reader needs. + """ + + __slots__ = ('user', 'source', 'target', 'path', 'code', 'reason', 'subject', + 'source_scope', 'target_scope') + + def __init__(self, user, source, target, path, + code=None, reason='', subject='', source_scope=None, target_scope=None): + self.user, self.source, self.target, self.path = user, source, target, list(path or ()) + self.code, self.reason, self.subject = code, reason, subject + self.source_scope, self.target_scope = source_scope, target_scope + + @property + def permitted(self): + """May this pivot be served at all? False exactly when `refusal()` has something to say.""" + return self.code is None + + def refusal(self): + """C10's `refusal` value: `{reason, subject}`, or None when the pivot is permitted. + + `subject` is the DATABASE KEY that refused, so a client can say which side of the join + stopped it instead of blaming the pivot as a whole. Two refusals have no key to give and + must not borrow an innocent one: a side the request never named answers with that side's + name (`'source'` / `'target'`), and a pivot with no identifiable principal answers + `'session'`. Naming a database in either case would point the reader at the wrong thing to + go fix. + """ + if self.code is None: + return None + return {'reason': self.reason, 'subject': self.subject} + + def limits(self, st=None): + """⭐ STANDING RULE 1's SECOND SENTENCE on this path: every `limit_report()` in force on + either side, as a list, in one vocabulary and no second one. + + ⛔ THIS WALL APPLIES NO CEILING OF ITS OWN AND NEVER WILL. A pivot narrows rows by + PERMISSION, and a permission answer that silently also truncated would understate every + related-record count it feeds while looking complete. What it does owe is the report for a + ceiling the DATABASES themselves carry, so a client renders the same limit sentence here + as on the grid rather than learning a pivot-shaped dialect ([[one-question-two-normalizers]]). + + ⚠ ASKED ONLY OF THE `ut_` NAMESPACE, AND THAT GUARD IS LOAD-BEARING, not an optimisation. + `user_tables.row_limit` resolves a registry topic key through `materialises()` -> + `get(key)` -> `{}` -> "materialised, not connected" and answers `MAX_ROWS`, so asking it + about `customer_data` INVENTS an editable-substrate ceiling for a compiled pool that has + none, and pays a whole-document copy to do it. + + Each report is `limit_report`'s own four keys plus `database`, because `subject` there is + always `'rows'` and a two-database answer must still say which side it is about. + """ + ut, out, seen = _ut(), [], set() + for key in self.path: + k = str(key or '') + if not k.startswith(ut.KEY_PREFIX) or k in seen: + continue + seen.add(k) + try: + report = ut.limit_report(k, st=st) + except Exception: # noqa: BLE001 + continue + if report: + out.append(dict(report, database=k)) + return out + + def envelope(self, st=None): + """The C10 response keys THIS wall owns, ready to merge into the route's own + `{rows, fields, sourceCount, targetCount}`. + + `path` always; `refusal` only when there IS one, because C10 spells it `refusal?` and a + `null` is not what an optional key means; `limits` only when something is reported, so the + ordinary answer is exactly the shape C10 names and it grows only when standing rule 1 has + something to say. + """ + env = {'path': list(self.path)} + refused = self.refusal() + if refused is not None: + env['refusal'] = refused + reported = self.limits(st=st) + if reported: + env['limits'] = reported + return env + + def source_grid(self, fields, rows, ctx=None, st=None): + """`(fields, rows)` of the SOURCE database, as this account may receive them.""" + return self._side(self.source, fields, rows, ctx, st) + + def target_grid(self, fields, rows, ctx=None, st=None): + """`(fields, rows)` of the TARGET database, as this account may receive them. + + ⛔ THIS IS THE SIDE THAT WOULD OTHERWISE DEFAULT OPEN. The source rows arrive already + walled, because the caller was looking at that grid; the target's rows are fetched BY the + pivot, for a database the reader may never have opened, and serving them unfiltered is the + whole hole. Same stored wall, same evaluator, same verdict `apply_row_scope` reaches at + every other door: a reader with a row filter on the target sees only their own rows here. + """ + return self._side(self.target, fields, rows, ctx, st) + + def _side(self, key, fields, rows, ctx, st): + """ONE narrower behind both sides, so the two cannot drift into different verdicts. + + ⛔⛔ BOTH WIRES COME OFF ONE CALL AND THAT IS WHY THIS RETURNS A PAIR. An earlier draft of + this method returned rows alone and told the caller to take the columns from + `scoped_fields`. That is the D-470 shape, rebuilt one wave after it was measured: the + field list arrives narrowed, the rows arrive whole, and the hidden value sits in the + payload under a column the reader cannot see — `scoped_fields()` declaring `['dba']` + beside a row carrying `custom_region_qa: 'TOP-SECRET-VALUE'`. `strip_row`'s own note is + the rule and a docstring is not enforcement, so the two wires are one return value and + cannot be taken apart by a caller in a hurry. + + Same order as `_scoped`, deliberately: `hidden_keys` -> `apply_row_scope` -> strip. ⛔ The + ROW wall is evaluated against the UNSTRIPPED contract, because a permanent filter may name + a column the reader is not allowed to SEE and dropping that predicate would WIDEN the read + rather than narrow it. And `st` goes to both halves or to neither (`visible_fields`' own + warning): lend it to one and the field list narrows by more than the rows did. + + ⛔⛔ `fields` MUST DECLARE THE USER-GENERATED COLUMNS, AND `st` MUST BE LENT. Wave-40 lane + C measured this on `apply_row_scope`: a wall naming a custom column narrows correctly only + when the field list DECLARES that column, and passing the STATIC contract instead makes + the same wall deny EVERY row (`[1]` vs `[]`, measured at `routes_customers.py` and + `routes_products.py`). On this door that failure wears the pivot's own worst costume: a + reader who may see their rows gets none, reads it as "no related records", and the wall + looks like the data. Hand this the assembled contract, not the compiled-in one. + + ⚠ THE THIRD WIRE IS `visible_overlays` AND IT IS NOT SERVED HERE. A pivot answers with + related records, not with the workspace overlay stratum; a route that ever adds overlays + to this response owes that call too, for the reason D-427 records. + + ⛔ RAISES ON A REFUSED PIVOT rather than returning `[]`. An empty list here would rebuild, + one layer down and inside the server where no client could tell the difference, the exact + "no related records" misreading `refusal` exists to prevent. + """ + if not self.permitted: + raise Denied(self.reason) + cols = list(fields or ()) + hide = hidden_keys(self.user, key, cols, st=st) + kept = apply_row_scope(rows, self.user, key, cols, ctx, st=st) + if not hide: + return cols, kept + return (visible_fields(cols, self.user, key, st=st), + [strip_row(r, hide) for r in kept]) + + +def pivot_scope(user, source, target, st=None, path=None): + """⭐⭐ CONTRACT C10 / RULING R12 — a pivot's row scope, decided on BOTH SIDES of the join. + + sc = perm_scope.pivot_scope(session.user, req.source, req.target, st=rt) + if not sc.permitted: + return {'rows': [], 'fields': [], 'sourceCount': 0, 'targetCount': 0, + **sc.envelope(st=rt)} + fields, rows = sc.target_grid(target_fields, related_rows, st=rt) + + Returns a `PivotScope` and NEVER raises for a permission answer, because C10 puts the refusal + ON THE WIRE beside the empty rows rather than throwing a status the client has to interpret. + Ask a refused verdict for rows anyway and you get `Denied`. + + THE ORDER, and every step of it fails closed: + + 1. **Both sides must be named.** A pivot missing one side is not a narrower pivot. + 2. **Both keys must be a legal relational endpoint** — `user_tables.is_linkable_target`, + C5's ONE answer, landed in this tree by W41-T15. Asked of BOTH sides rather than only the + target: a join has two endpoints, and `users`, `object_shares`, `cohort` and the archived + `customer`/`product` keys are no more joinable as a source than as a target. + 3. **R12: the path is exactly `PIVOT_PATH_LEN` databases.** A longer `path`, or one that + disagrees with `(source, target)`, is refused as a chain. + 4. **`may_read` on the SOURCE, then `may_read` on the TARGET** — C1's IF question, asked + twice, composed and never re-derived. Source first, so the `subject` a customers-only + reader gets back names the TARGET that actually stopped them. + 5. **`derive_pool_scope` on BOTH**, once the verdict is a pass, so the pool behind each side + is BUILT with the scope that side's permanent filter derives. Amendment 3's argument is + not weaker across a join, it is stronger: a target pool built consolidated hands a + Fisch-only reader products whose revenue is Fisch+Royal, and the row list looks perfectly + correct while every number on it is somebody else's. + + ⛔ AN EXCEPTION OUT OF EITHER `may_read` IS A REFUSAL — never a crash, and never a pass. + `may_read` reaches the store on a `ut_*` key, and a store outage on the target side must not + be the one condition under which the target wall is skipped. The reason is FIXED COPY: an + exception's text is a server detail and this string reaches a screen. + + ⛔⛔ AND A MISSING PRINCIPAL IS A REFUSAL BEFORE ANYTHING ELSE IS ASKED — MEASURED, not + theorised. `may_read(None, 'customer_data')` answers **True** today: `_rec(None)` is `{}`, so + the record reads as UN-MIGRATED and the legacy `perms.may_open` wall admits it, which is the + documented fail-open window `perms_v` exists to close. Every other door survives that because + it uses the principal ONCE and then serves a pool the caller already scoped. THIS door hands + the principal forward: `target_grid` would carry the `None` into `apply_row_scope`, find no + entry, find no filter, and return the target database WHOLE. So a route that lost its session + would not 401, it would pivot. `scoped_table`'s own rule stated one layer up — *"the only + thing a defaulted principal could mean is 'unscoped', which is the widening direction"* — and + this is the door with the most to lose by it. + """ + src, tgt = str(source or '').strip(), str(target or '').strip() + walk = [str(p or '').strip() for p in path] if path is not None else [src, tgt] + + def refuse(code, reason, subject): + return PivotScope(user, src, tgt, walk, code, reason, subject) + + if not isinstance(user, dict) or not user: + return refuse('unresolvable', + 'This session could not be identified, so the pivot refuses rather than ' + 'serve records it cannot scope. Sign in again and reopen it.', + 'session') + + if not src or not tgt: + return refuse('no_database', + 'A pivot needs a source database and a target database, and this request ' + 'names only one.', + 'source' if not src else 'target') + + ut = _ut() + for key in (src, tgt): + if not ut.is_linkable_target(key): + return refuse('not_relational', + f"'{key}' is not a database a pivot can join. Choose a linked database " + f'on both sides of the pivot.', + key) + + # R12, refused BY NAME so the message can point at the hop that is not supported instead of at + # the pivot the reader did ask for. + if len(walk) > PIVOT_PATH_LEN: + extra = next((k for k in walk[PIVOT_PATH_LEN:] if k), tgt) + return refuse('chained', + f'A pivot joins exactly two databases, and this one names {len(walk)}. ' + f"Open '{extra}' from the pivoted records instead of chaining to it here.", + extra) + if walk != [src, tgt]: + return refuse('chained', + 'A pivot joins exactly two databases, and the path it was given is not the ' + 'two this request names. Reopen the pivot from the database you are ' + 'looking at.', + tgt) + + for key, side in ((src, 'source'), (tgt, 'target')): + try: + allowed = bool(may_read(user, key, st=st)) + except Exception: # noqa: BLE001 + return refuse('unresolvable', + f"'{key}' could not be resolved just now, so this pivot refuses rather " + f'than serve records it cannot scope. Try again in a moment.', + key) + if allowed: + continue + if side == 'target': + reason = (f"This account is not permitted to read '{key}', so its records are " + f'withheld here. This is a permission refusal, not a database with nothing ' + f'related in it.') + else: + reason = (f"This account is not permitted to read '{key}', so a pivot cannot start " + f'from it.') + return refuse('unreadable', reason, key) + + return PivotScope(user, src, tgt, walk, + source_scope=derive_pool_scope(user, src), + target_scope=derive_pool_scope(user, tgt)) diff --git a/platform/core/script_sandbox.py b/platform/core/script_sandbox.py index 231ce07dd952f2d67caf31e11e9f1525a6a5a38f..babf60594d17d540e348efa836e40056260a1f96 100644 --- a/platform/core/script_sandbox.py +++ b/platform/core/script_sandbox.py @@ -1,519 +1,519 @@ -"""core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python. - -Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they -want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this -script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was -stated, so this file is the guardrail, and one engine serves both items. - -════════════════════════════════════════════════════════════════════════════════════════════════ -⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE. - -In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated -namespace stops import, file, network and environment access — but it **cannot cap memory and -cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow -request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**: -`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the -allow-list inside. Neither half is sufficient; both are load-bearing. - -⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under -the CALLING user's record and serialises the result; the child imports nothing from this repo and -holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store -path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child -reports its own `sys.modules`, and no `core.*` name may appear in it. - -⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute -names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and -the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute -lookup inside `format`, so there is no `ast.Attribute` node to refuse). -════════════════════════════════════════════════════════════════════════════════════════════════ - -The two layers, and they refuse DIFFERENT things on purpose: - - 1. `check_source()` — a pure function over source text. Refuses a construct the language offers - and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every - attribute name outside `ALLOWED_ATTRS`. - 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`, - `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets - past layer 1 still finds nothing to call. - -⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is -refused twice, so a negative control that drops ONE layer sees the other refuse and reports -green — the shape that already cost this wave one missed control in `routes_agent_harness`. So -each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings, -and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching -boundary goes red. -""" -import ast -import json -import os -import subprocess -import sys -import tempfile -import time -from pathlib import Path - -#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform. -DEFAULT_TIMEOUT_S = 10.0 - -#: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report. -DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024 - -#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the -#: wall-clock kill is the primary control and this is the backstop for a child that stops being -#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal. -DEFAULT_CPU_SECONDS = 15 - -#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer, -#: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything -#: that escaped far enough to write to fd 1 has nowhere for it to land. -MAX_STDOUT_BYTES = 64 * 1024 - -#: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule -#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()` -#: returns a named limit carrying its cause and a recommendation. -MAX_PAYLOAD_BYTES = 32 * 1024 * 1024 - -#: The emitted spec. A render spec is a description of a picture; one larger than this is data -#: pretending to be a description. -MAX_SPEC_BYTES = 2 * 1024 * 1024 - -MAX_SOURCE_BYTES = 128 * 1024 - - -# ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════ -#: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` / -#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its -#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__` -#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every -#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never -#: awaited is a silent no-op that looks like a working script). -ALLOWED_NODES = frozenset(""" -Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise -If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword -BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name -Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension -Load Store Del -And Or Not Invert UAdd USub -Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult -Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn -""".split()) - -#: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is -#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are -#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace, -#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__` -#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this -#: sandbox has not been asked for. -#: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup -#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are -#: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused. -ALLOWED_ATTRS = frozenset(""" -append extend insert pop remove clear sort reverse copy count index -keys values items get setdefault update -add discard union intersection difference issubset issuperset -join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold -replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix -isdigit isalpha isalnum isspace isupper islower isnumeric -real imag numerator denominator -""".split()) - - -class Refused(Exception): - """A named refusal: `code` for a caller to branch on, `message` for a person to read.""" - - def __init__(self, code, message): - self.code, self.message = code, message - super().__init__(f"{code}: {message}") - - -def _attr_ok(name): - """An attribute name passes only if it is on the list AND is not private. - - ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every - private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by - accident, which is the one edit a future reader is most likely to make in a hurry. - """ - return name in ALLOWED_ATTRS and not name.startswith("_") - - -def check_source(source): - """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`. - - ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no - process and touches no store, so a gate can hand it a hundred hostile strings for free and an - NC can drop one entry from one frozenset and watch exactly this function change its answer. - """ - text = str(source or "") - if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - return Refused("source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source") - try: - tree = ast.parse(text) - except SyntaxError as exc: - return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}") - - for node in ast.walk(tree): - kind = type(node).__name__ - if kind not in ALLOWED_NODES: - return Refused("refused_construct", - f"line {getattr(node, 'lineno', 0)}: this sandbox does not run " - f"{_english(kind)}") - if isinstance(node, ast.Attribute) and not _attr_ok(node.attr): - return Refused("refused_attribute", - f"line {getattr(node, 'lineno', 0)}: the attribute " - f"'{node.attr}' is not available inside a script view") - # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a - # script that could bind one could shadow the machinery it runs on top of. - if isinstance(node, ast.Name) and node.id.startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: names starting with an " - f"underscore are reserved by the sandbox") - if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str( - getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: names starting with an " - f"underscore are reserved by the sandbox") - if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with " - f"an underscore are reserved by the sandbox") - return None - - -_ENGLISH = { - "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition", - "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop", - "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield", - "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement", - "Match": "a match statement", -} - - -def _english(kind): - return _ENGLISH.get(kind, f"a {kind} expression") - - -# ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════ -#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child. -#: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input` -#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help` -#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step -#: on a published escape, and none has ever been asked for by a script that shapes rows. -#: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause -#: cannot name what it catches is a construct that reads as supported and is not. They are safe -#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this -#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway. -SANDBOX_BUILTIN_NAMES = ( - "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex " - "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed " - "round set slice sorted str sum tuple zip True False None " - "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError " - "AttributeError StopIteration OverflowError" -).split() - -#: The literal program the child runs. It is TEXT rather than a module because the child must -#: import nothing from this repo: a module would be found on `sys.path` and would drag `core` -#: with it, which is exactly the second store path W1 forbids. -#: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so -#: the runner's own machinery cannot be shadowed by the source it executes. -_RUNNER = r''' -import json as _json, os as _os, sys as _sys - -_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read()) -_out = {"ok": False, "code": "not_run", "message": "the script did not run", - "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}} - -# ── the caps this platform can actually apply, reported either way (standing rule 1) ────────── -try: - import resource as _res - _mem = int(_pay["memoryBytes"]) - _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem)) - _out["caps"]["memory"] = True - _cpu = int(_pay["cpuSeconds"]) - _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu)) - _out["caps"]["cpu"] = True -except Exception: - # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says - # which of the three held, never a silent partial. - pass - -_printed = [] -_spent = [0] -_LIMIT = int(_pay["maxStdout"]) - - -def _print(*_a, **_k): - _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n") - _room = _LIMIT - _spent[0] - if _room > 0: - _printed.append(_text[:_room]) - _spent[0] += len(_text) - - -class _Refusal(Exception): - """The SANDBOX refusing, as distinct from the SCRIPT failing. - - Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the - script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's - own code. Two different facts, two different codes. - """ - - -_emitted = [] - - -def _emit(_spec): - if not isinstance(_spec, dict): - raise _Refusal("emit() takes a view spec, which is a dictionary") - if _emitted: - raise _Refusal("emit() was already called; a script view emits exactly one view") - _emitted.append(_spec) - - -_rows = _pay["rows"] -_fields = _pay["fields"] -_bound = _pay["table"] - - -def _scoped_table(_table=None): - if _table is not None and str(_table) != _bound: - raise _Refusal( - "this script view is bound to the database '" + _bound + "' and asked for '" - + str(_table) + "'. A script view reads its own database only") - return [dict(_r) for _r in _rows] - - -def _scoped_fields(): - return [dict(_f) for _f in _fields] - - -_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict) - else getattr(__builtins__, _n) - for _n in _pay["builtins"]}} -_ns["__builtins__"]["print"] = _print -_ns["print"] = _print -_ns["emit"] = _emit -_ns["scoped_table"] = _scoped_table -_ns["scoped_fields"] = _scoped_fields -_ns["table"] = _bound - -try: - exec(compile(_pay["source"], " - - + + + + + + + Loopable + + + +
+
+ + + diff --git a/web/src/App.tsx b/web/src/App.tsx index 8e740b30aa739af89fe5978458176d8cc964a6e6..0dcffc3f4790d9b8a0842285b64f4ff08b9d5da4 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,41 +1,41 @@ -import { Suspense, lazy } from "react"; -// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 -// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the -// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build -// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, -// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. -// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is -// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is -// indistinguishable from a broken one. -const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); -import { OverlayProvider } from "./customer-grid/OverlaySurface"; -import { isStreamlitComponent } from "./customer-grid/hostBridge"; -import Shell from "./shell/Shell"; - -// HOSTED = either Streamlit path (the component iframe, or the legacy injected -// payload). There the surrounding app owns all chrome and this tree renders the -// BARE grid, byte-for-byte the behavior the embed always had. Only the -// standalone build gets the shell (wave 4, the strangler frame). Computed once -// at module level, like useCustomerData's own mode constants — the host mode -// cannot change within a page lifetime. -// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to -// install the standalone event sink, and two copies of a mode predicate is how -// the two halves of "standalone-only" end up disagreeing. -export const HOSTED = - isStreamlitComponent() || - typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; - -export default function App() { - if (HOSTED) { - return ( -
- - - -
- ); - } - return ; -} +import { Suspense, lazy } from "react"; +// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 +// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the +// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build +// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, +// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. +// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is +// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is +// indistinguishable from a broken one. +const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); +import { OverlayProvider } from "./customer-grid/OverlaySurface"; +import { isStreamlitComponent } from "./customer-grid/hostBridge"; +import Shell from "./shell/Shell"; + +// HOSTED = either Streamlit path (the component iframe, or the legacy injected +// payload). There the surrounding app owns all chrome and this tree renders the +// BARE grid, byte-for-byte the behavior the embed always had. Only the +// standalone build gets the shell (wave 4, the strangler frame). Computed once +// at module level, like useCustomerData's own mode constants — the host mode +// cannot change within a page lifetime. +// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to +// install the standalone event sink, and two copies of a mode predicate is how +// the two halves of "standalone-only" end up disagreeing. +export const HOSTED = + isStreamlitComponent() || + typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; + +export default function App() { + if (HOSTED) { + return ( +
+ + + +
+ ); + } + return ; +} diff --git a/web/src/account/SubscriptionPage.tsx b/web/src/account/SubscriptionPage.tsx index 59f00a708e4ab698bd6e9863238898e970f0a56c..2cbc71a05dca231d01b3badbb396d5bd70452472 100644 --- a/web/src/account/SubscriptionPage.tsx +++ b/web/src/account/SubscriptionPage.tsx @@ -1,49 +1,49 @@ -// --------------------------------------------------------------------------- -// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). -// -// Owner item 13, verbatim: *"add below settings the following button: Feedback …, -// Usage credits …, and a Subscription module … Specialized for the Royal Imports -// tenant, do not show it."* -// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the -// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. -// -// This is the third of those three, and the only one with nothing behind it yet. -// So it says so, in the words this app already uses for a boarded door -// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later -// release."). One voice for one state — DESIGN.md 1's "variety is a defect". -// -// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. -// C6 puts the Royal Imports rule in TWO places that are both the frame's: the -// account MENU does not draw the row, and the ROUTE refuses. A third opinion here -// would be a third thing to keep in step, and the day they disagree the reader -// gets a menu row that opens a page which says it does not exist. The page renders -// the same for every tenant that can reach it, because reaching it IS the decision. -// -// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. -// The wave's rule is that a prop must be REQUIRED, never optional — an optional one -// degrades to "the page does not exist", which is indistinguishable from never -// built. That rule constrains the props that EXIST; it does not ask for a prop that -// carries nothing. There is nothing this page needs from the frame: it holds no -// state, makes no call, and takes no decision the frame has already taken. A -// `tenant` prop here would be exactly the second opinion the paragraph above -// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. -// --------------------------------------------------------------------------- - -import "./account.css"; - -export default function SubscriptionPage() { - return ( -
-

Subscription

-
- {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not - narrate itself). The reader arrived here on purpose and wants one fact: is there - anything to do? There is not, and every further word would be spent on someone - who is already leaving. No feature list, no roadmap, no "in the meantime". */} -

- Under construction. Plans, billing and invoices will open here in a later release. -

-
-
- ); -} +// --------------------------------------------------------------------------- +// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). +// +// Owner item 13, verbatim: *"add below settings the following button: Feedback …, +// Usage credits …, and a Subscription module … Specialized for the Royal Imports +// tenant, do not show it."* +// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the +// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. +// +// This is the third of those three, and the only one with nothing behind it yet. +// So it says so, in the words this app already uses for a boarded door +// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later +// release."). One voice for one state — DESIGN.md 1's "variety is a defect". +// +// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. +// C6 puts the Royal Imports rule in TWO places that are both the frame's: the +// account MENU does not draw the row, and the ROUTE refuses. A third opinion here +// would be a third thing to keep in step, and the day they disagree the reader +// gets a menu row that opens a page which says it does not exist. The page renders +// the same for every tenant that can reach it, because reaching it IS the decision. +// +// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. +// The wave's rule is that a prop must be REQUIRED, never optional — an optional one +// degrades to "the page does not exist", which is indistinguishable from never +// built. That rule constrains the props that EXIST; it does not ask for a prop that +// carries nothing. There is nothing this page needs from the frame: it holds no +// state, makes no call, and takes no decision the frame has already taken. A +// `tenant` prop here would be exactly the second opinion the paragraph above +// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. +// --------------------------------------------------------------------------- + +import "./account.css"; + +export default function SubscriptionPage() { + return ( +
+

Subscription

+
+ {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not + narrate itself). The reader arrived here on purpose and wants one fact: is there + anything to do? There is not, and every further word would be spent on someone + who is already leaving. No feature list, no roadmap, no "in the meantime". */} +

+ Under construction. Plans, billing and invoices will open here in a later release. +

+
+
+ ); +} diff --git a/web/src/account/UsagePage.tsx b/web/src/account/UsagePage.tsx index 6ab2ceb32d8c3e1213c3df4e9667466d4384d916..f735d26732cc9caca5ada0eb71c190c9584ec1b4 100644 --- a/web/src/account/UsagePage.tsx +++ b/web/src/account/UsagePage.tsx @@ -1,297 +1,297 @@ -// --------------------------------------------------------------------------- -// account/UsagePage.tsx — WAVE 35 · W35-T19 (owner item 13, ruling R9, C6/C7). -// -// R9: ONE meter for every AI surface. It REPORTS weekly usage against an -// allowance and does not cut anyone off this wave; over the allowance the -// product still works and the bar is red. -// -// ⛔⛔ THIS PAGE MAY NOT INVENT A FIGURE, AND THAT IS THE WHOLE TICKET. -// Every number here comes off `GET /usage` — the week, the reset date, the -// allowance, each surface's tokens and calls. Nothing is derived from a clock, -// nothing is estimated, and a surface with no ledger lines shows **0**, not a -// blank and not an omission (E-9 guarantees all four surfaces are always -// present, so there is no missing-row branch to get wrong). -// [[no-unverifiable-aggregates]] -// -// ⛔ AND `unmeasured` IS RENDERED, NOT SWALLOWED. Those are real calls whose -// provider declined to report a token count, so the total beside them is a -// FLOOR. A page that hides them presents the floor as a complete figure, which -// is the cost-surprise failure one step removed — the same reason -// `usage_ledger` books an unmeasured call as unknown rather than as zero. -// -// ⚠ SURFACE/READ SPLIT, as on Home and Starred: `renderToStaticMarkup` runs no -// effect, so a shot of the fetching component photographs only its spinner. -// --------------------------------------------------------------------------- - -import { useCallback, useEffect, useRef, useState } from "react"; - -import { fmt } from "../ui/fmt"; -import { loadUsage } from "./accountApi"; -import type { Usage, UsageSurface } from "./accountApi"; -import "./account.css"; - -/** - * The four surfaces C7 names, in the order R9 lists them. - * - * ⚠ AN UNKNOWN KEY IS HUMANISED, NEVER DROPPED. Dropping it would hide real usage from a meter - * whose whole job is to account for it, and printing the raw key would put an internal identifier - * on screen. The day a fifth LLM entry point lands (this product added two in one wave) it shows - * up here as a readable name with the right number beside it, and somebody can then decide what - * to call it. - */ -const SURFACE_LABEL: Record = { - assistant: "Assistant", - field_agent: "Field agents", - ai_review: "Reviews", - automation_draft: "Drafting", -}; -const ORDER = ["assistant", "field_agent", "ai_review", "automation_draft"]; - -export function surfaceLabel(key: string): string { - const known = SURFACE_LABEL[key]; - if (known) return known; - const words = key.replace(/[_-]+/g, " ").trim(); - return words ? words.charAt(0).toUpperCase() + words.slice(1) : key; -} - -/** C7's order first, then anything the server added, alphabetically. */ -export function orderSurfaces(surfaces: UsageSurface[]): UsageSurface[] { - const rank = (s: UsageSurface) => { - const i = ORDER.indexOf(s.surface); - return i === -1 ? ORDER.length : i; - }; - return [...surfaces].sort( - (a, b) => rank(a) - rank(b) || surfaceLabel(a.surface).localeCompare(surfaceLabel(b.surface)) - ); -} - -/** - * The bar's fill, as a percentage, CLAMPED to 100. - * - * ⚠ The clamp is what keeps "over the allowance" a COLOUR rather than a bar that overflows its - * own track and paints across the page. The number above the bar is unclamped and is where the - * overage is actually read. - * ⚠ An allowance of 0 (unset) yields 0 rather than a division by zero: no allowance means the - * meter has nothing to measure against, and a full red bar would be an assertion nobody made. - */ -export function barPct(value: number, of: number): number { - if (!(of > 0)) return 0; - return Math.max(0, Math.min(100, (value / of) * 100)); -} - -export type UsageLoad = - | { phase: "loading" } - | { phase: "ready"; usage: Usage } - | { phase: "failed"; message: string }; - -function Bar({ pct, over }: { pct: number; over: boolean }) { - return ( - {queryCitationLabel(citation)}; -} - -/** One glyph, drawn once and turned over for the other. R20 asked for thumbs; these are thumbs. */ -function ThumbIcon({ down = false }: { down?: boolean }) { - return ( - - ); -} - -export function AssistantMessage({ message, view, citations, onPreview, onRate }: { - message: QueryMessage; view?: SavedQuery; citations: QueryCitation[]; onPreview: (id: string) => void; - /** Optional so the message can be rendered outside a live conversation without a rating door. */ - onRate?: (id: string, rating: "up" | "down" | null, reason?: string) => void; -}) { - const [openSources, setOpenSources] = useState(false); - const [askWhy, setAskWhy] = useState(false); - const [why, setWhy] = useState(""); - const matching = citations.filter((citation) => (message.citationIds || []).includes(citation.id)); - if (message.role === "user") { - return ( -
-

{message.content}

-
- ); - } - const numeric = message.numeric; - return ( -
-

{message.content}

- {numeric && numeric.value !== null && numeric.value !== undefined ? ( -

- {numeric.label} - {numeric.value.toLocaleString()} -

- ) : null} - {view ? ( - - ) : null} - {/* ⚠ The thumbs are NOT a survey: the server reads a thumbs-down back into the next turn of - this thread. That is why a down asks for one short reason and why both are clearable. */} - {onRate ? ( -
- - - {message.rating === "down" && !askWhy && message.ratingReason ? ( - {message.ratingReason} - ) : null} -
- ) : null} - {onRate && askWhy && message.rating === "down" ? ( -
{ - event.preventDefault(); setAskWhy(false); onRate(message.id, "down", why.trim()); - }}> - setWhy(event.currentTarget.value)} /> - -
- ) : null} - {matching.length ? ( -
- - {openSources ? ( -
- {matching.map((citation) => )} -
- ) : null} -
- ) : null} -
- ); -} - -function initialIndex(): QueryIndex { - return { threads: [], messages: [], views: [], citations: [], models: ["auto"], - modelStatus: [], sources: [] }; -} - -/** - * ⭐⭐ OWNER ITEM 2 (2026-08-15) — A PLAIN CHATBOT, IN THREE BORROWED SHAPES. - * - * Verbatim: *"AI assistant module should be a plain chatbot interface where the user can choose - * which model to use… I really like the plain look for Glean… Notice how the database you can - * select is at the bottom there with all the logos, that's how we should allow user to select - * databases too… The interface should exactly be like ChatGPT… ALSO I want a navigation for the - * chat EXACTLY like how Airtable does it."* - * - * So the surface is assembled from the three references the owner supplied, each for the part it - * is actually about: - * · `reference/ChatGPT 1.png` — the CONVERSATION: one centred column, the question as a bubble - * on the right, the answer as plain flowing text with no label and no bubble, the composer - * pinned to the bottom with its controls INSIDE the field. - * · `reference/Glean 1.png` — the SOURCES: a row of chips carrying each source's own mark, - * directly under the composer. Not a dropdown, not a settings panel. - * · `reference/Airtable AI 1.png` — the NAVIGATION: "New chat" at the top of a left panel, the - * thread list under it, and a centred "How can I help?" when the thread is empty. - * - * ⛔ WHAT IS DELIBERATELY NOT COPIED: ChatGPT's dark canvas and Glean's vendor logos. The palette - * is Loopable's own (`DESIGN.md`, and the font-token rule is gated app-wide over every CSS file - * in this tree), and the chips wear `FolderMark` — the SAME mark the nav rail draws for that - * database — because a brand logo for someone else's product is what those references have and - * this product's databases are not brands. "Exactly like" is about the interaction shape. - * - * ⚠ AND THE TOOL BOUNDARY IS NARROWER THAN THE SENTENCE "the chatbot can basically call the tools - * in our App". Under ruling R1 the assistant operates the caller's permitted database/view tools - * and nothing else: server-side there is exactly ONE tool, `build_view`. It cannot run an - * automation, touch a connector, or change a setting, and no part of this page implies it can. - */ -export default function AssistantPage({ granted }: AssistantPageProps) { - const [index, setIndex] = useState(initialIndex); - const [activeThread, setActiveThread] = useState(""); - const [selected, setSelected] = useState([]); - const [target, setTarget] = useState(""); - const [model, setModel] = useState("auto"); - const [question, setQuestion] = useState(""); - const [busy, setBusy] = useState(false); - const [problem, setProblem] = useState(""); - const [loaded, setLoaded] = useState(false); - const [confirmThread, setConfirmThread] = useState(""); - const [mode, setMode] = useState(readMode); - const [activeQuery, setActiveQuery] = useState(""); - /** Bumped on delete so the hosted workspace's own copy of the list cannot go stale behind us. */ - const [queryRefresh, setQueryRefresh] = useState(0); - const openCancel = useRef<(() => void) | null>(null); - const foot = useRef(null); - /** A counter, not a random id: the optimistic message only has to be unique within this page. */ - const pending = useRef(0); - - const databases = useMemo(() => databaseEntries(granted).filter((entry) => entry.kind !== "group"), [granted]); - const labels = useMemo(() => new Map(databases.map((entry) => [entry.key, entry.label])), [databases]); - /** - * ⭐⭐ WHICH CHIPS CAN ACTUALLY BE ASKED, and why the ones that cannot say so. - * The nav grants twelve databases on tenant #0 and the read boundary can answer for two: the - * ten `ut_odoo_*` grids are served through the connector mirror, which the Assistant refuses by - * design. Offering ten doors that cannot open — and letting the reader discover it one spent - * prompt at a time — is the same defect shape as a rail that paints two rows differently from - * five. ⚠ Absent from the map ⇒ TREATED AS ANSWERABLE: the server is the authority and its - * refusal is still the wall; a client that greyed out everything it had not heard about would - * hide a working source the moment this field failed to arrive. - */ - const sourceStatus = useMemo( - () => new Map(index.sources.map((row) => [row.database, row])), [index.sources]); - const blockedReason = useCallback( - (key: string) => (sourceStatus.get(key)?.answerable === false - ? sourceStatus.get(key)?.reason || "this database cannot be asked yet" : ""), - [sourceStatus]); - const messages = index.messages.filter((row) => row.threadId === activeThread); - const viewById = useMemo(() => new Map(index.views.map((view) => [view.id, view])), [index.views]); - - /** Offered is not callable: a model whose key this deployment does not hold says so, and why. */ - const modelBlocked = useCallback((key: string) => { - const row = index.modelStatus.find((item) => item.model === key); - return row && row.available === false - ? row.reason || "this model is not available on this deployment" : ""; - }, [index.modelStatus]); - - const reload = useCallback(async () => { - const result = await fetchQueries(); - setLoaded(true); - if (result.ok) { - setIndex(result.value); - // ⚠ Falls back to Auto when the SELECTED model is offered but not callable, not only when it - // has left the list: a stored thread can restore a model whose key was removed since, and a - // disabled option the picker cannot clear is a control that refuses its own value. - const callable = (key: string) => result.value.models.includes(key) - && result.value.modelStatus.every((row) => row.model !== key || row.available); - setModel((current) => callable(current) ? current : "auto"); - } - }, []); - - useEffect(() => { void reload(); }, [reload]); - useEffect(() => () => openCancel.current?.(), []); - useEffect(() => { - const permitted = databases.map((entry) => entry.key); - setSelected((current) => current.filter((key) => permitted.includes(key))); - setTarget((current) => permitted.includes(current) ? current : ""); - }, [databases]); - // The conversation reads bottom-up, like every chat the owner named. - useEffect(() => { foot.current?.scrollIntoView({ block: "end" }); }, [messages.length, busy]); - - /** - * The `?mode=` the redirect carried has been read into state; drop it so a toggle made later is - * not overridden by a stale parameter on the next reload. `replaceState` does NOT fire - * `hashchange`, so the shell never re-resolves the route and nothing flickers. - * - * ⛔ AND IT IS STORED IN THE SAME BREATH, WHICH IS THE HALF THAT WAS MISSING. `useState(readMode)` - * reads the hash and writes NOTHING; only `pickMode` persists. So a reader who arrived in Query - * by following a `#/query` link, never touching the toggle, would have the parameter stripped - * here and then land back in Chat on the next reload, restored from a choice made days earlier. - * Following the link IS choosing. - * - * ⛔⛔ AND IT LISTENS FOR `hashchange`, WHICH IS THE DEFECT THE OWNER REPORTED (QA 2026-08-17, - * measured on the deployed build). `#/query` and `#/assistant` resolve to the SAME route, so - * following a `#/query` link from anywhere inside the running app is a same-document hash - * change: this component is already mounted, `useState(readMode)` does not run a second time, - * and the panel stayed on Chat while the address bar read `?mode=query`. Only a full reload - * ever showed Query. **A mount-time read cannot see an arrival that does not remount** — the - * listener is the whole fix, and `arrivalMode` is the ONE parser both paths call so a later - * change cannot make the two disagree. - */ - useEffect(() => { - if (typeof window === "undefined") return; - const consume = () => { - const asked = arrivalMode(window.location.hash); - if (!asked) return; - setMode(asked); - storeMode(asked); - window.history.replaceState(null, "", `#/${ASSISTANT_ROUTE}`); - }; - consume(); - window.addEventListener("hashchange", consume); - return () => window.removeEventListener("hashchange", consume); - }, []); - - const pickMode = useCallback((next: AssistantMode) => { - setMode(next); storeMode(next); setProblem(""); - }, []); - - /** The saved Query views, grouped exactly as the standalone page groups them (ONE function). */ - const queryList = useMemo(() => queryGroups(index.views, databases), [index.views, databases]); - - const selectQuery = useCallback((qid: string) => { - setActiveQuery(qid); - openCancel.current?.(); - openCancel.current = openBuiltView(qid, true); - }, []); - - /** A renamed or duplicated artefact, merged in place, and the hosted workspace told to re-read. */ - const upsertQuery = useCallback((view: SavedQuery) => { - setIndex((current) => ({ - ...current, views: [view, ...current.views.filter((row) => row.id !== view.id)], - })); - setQueryRefresh((count) => count + 1); - }, []); - - const removeQuery = useCallback(async (qid: string) => { - const result = await deleteQuery(qid); - if (!result.ok) { setProblem(result.message); return; } - setIndex((current) => ({ ...current, views: current.views.filter((row) => row.id !== qid) })); - setActiveQuery((current) => (current === qid ? "" : current)); - setQueryRefresh((count) => count + 1); - }, []); - - /** Arriving in Query with nothing chosen selects the newest view of the first database. */ - useEffect(() => { - if (mode !== "query" || activeQuery) return; - const first = queryList[0]?.views[0]; - if (first) selectQuery(first.id); - }, [mode, activeQuery, queryList, selectQuery]); - - const newChat = useCallback(() => { - setActiveThread(""); setQuestion(""); setProblem(""); - }, []); - - /** Opening a past chat restores the sources and the model it ran with (R6). */ - const openThread = useCallback((id: string) => { - setActiveThread(id); setProblem(""); - const row = index.threads.find((thread) => thread.id === id); - if (!row) return; - const permitted = databases.map((entry) => entry.key); - const sources = (row.sources || []).filter((key) => permitted.includes(key)); - if (sources.length) { setSelected(sources); setTarget(sources[0]); } - if (row.model && index.models.includes(row.model) && !modelBlocked(row.model)) setModel(row.model); - }, [databases, index.models, index.threads, modelBlocked]); - - const toggleSource = useCallback((key: string) => { - if (blockedReason(key)) return; - setSelected((current) => { - const next = current.includes(key) ? current.filter((item) => item !== key) : [...current, key]; - setTarget((currentTarget) => next.includes(currentTarget) ? currentTarget : next[0] || ""); - return next; - }); - }, [blockedReason]); - - /** - * ⭐⭐ R16 — ENTER GOES TO THE CHAT, NOT TO A WAIT. Owner: *"Pressing enter goes immediately to - * the chat view showing that it is loading; there is a thinking animation."* - * - * This used to POST and only then touch `index`, so for the whole model call the reader sat on - * the opening screen looking at their own unsent sentence with nothing moving. The message is - * appended FIRST, which is what flips `empty` and hands the surface to the stream; `busy` then - * draws the app's one loading mark under it. - * - * ⛔ THE INVERSE IS CAPTURED BEFORE THE WRITE. An optimistic append that is not reconciled on - * failure leaves a message on screen the server never received, and the next turn replays it to - * the model as if it had [[undo-capture-before-the-write]]. The typed text goes back in the - * composer too: losing somebody's sentence to a failed request is the second injury. - */ - const ask = useCallback(async () => { - const text = question.trim(); - if (!text || !target || !selected.includes(target) || busy) return; - const pendingId = `pending_${(pending.current += 1)}`; - const optimistic: QueryMessage = { - id: pendingId, threadId: activeThread, role: "user", content: text, - createdAt: new Date().toISOString(), targetDatabase: target, - sources: selected, requestedModel: model, - }; - const withoutPending = (rows: QueryMessage[]) => rows.filter((row) => row.id !== pendingId); - setBusy(true); setProblem(""); setQuestion(""); - setIndex((current) => ({ ...current, messages: [...current.messages, optimistic] })); - const result = await submitChat({ question: text, database: target, sources: selected, - ...(activeThread ? { threadId: activeThread } : {}), model }); - setBusy(false); - if (!result.ok) { - setIndex((current) => ({ ...current, messages: withoutPending(current.messages) })); - setQuestion(text); - setProblem(result.message); - return; - } - setActiveThread(result.value.thread.id); - setIndex((current) => ({ - ...current, - threads: [result.value.thread, ...current.threads.filter((row) => row.id !== result.value.thread.id)], - // The server's own copy of the question replaces the optimistic one, so a thread never holds - // two of the same turn and the id the rest of the page keys on is the durable one. - messages: [...withoutPending(current.messages), result.value.userMessage, result.value.message], - views: result.value.view ? [result.value.view, ...current.views.filter((row) => row.id !== result.value.view?.id)] : current.views, - citations: [...current.citations, ...result.value.citations], - })); - }, [activeThread, busy, model, question, selected, target]); - - /** ⛔ The chat goes; the views it built STAY (they live in Query). Two-click, in the row. */ - const removeThread = useCallback(async (id: string) => { - const result = await deleteThread(id); - if (!result.ok) { setProblem(result.message); return; } - setIndex((current) => ({ - ...current, - threads: current.threads.filter((row) => row.id !== id), - messages: current.messages.filter((row) => row.threadId !== id), - })); - setActiveThread((current) => (current === id ? "" : current)); - setConfirmThread(""); - }, []); - - /** - * The rating goes to the server and the SERVER'S copy comes back, rather than the page patching - * its own guess: the reason is normalised and cleared there, and a rating that only looked - * applied is exactly the shape this module keeps being bitten by. - */ - const rate = useCallback(async (id: string, rating: "up" | "down" | null, reason?: string) => { - const result = await rateMessage(id, rating, reason); - if (!result.ok) { setProblem(result.message); return; } - setIndex((current) => ({ - ...current, - messages: current.messages.map((row) => (row.id === id ? result.value : row)), - })); - }, []); - - /** A built view opens WHERE THE READER ALREADY IS: the toggle flips, the list follows. */ - const preview = useCallback((qid: string) => { - pickMode("query"); - selectQuery(qid); - }, [pickMode, selectQuery]); - - const empty = messages.length === 0; - const composer = ( -
-
-