diff --git a/RELEASES.json b/RELEASES.json index f4e29d8c66e3863e13fe549266c5b3f8211d5bbb..311f1cf38304b6be59b510755206e110cad17569 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "55ced50", + "current": "c9ad659 (c9ad659)", "releases": [ + { + "version": "v29", + "sha": "55ced50", + "date": "2026-08-18", + "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)." + }, { "version": "v28", "sha": "2793e4b", diff --git a/VERSION b/VERSION index 9c05074ce23421506d058184a162d5e6a2d47210..a328161406df0015dbed950205a581c91a4a46d1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -55ced50 +c9ad659 (c9ad659) diff --git a/api/ai_review.py b/api/ai_review.py index 29f16c1b0d27c10df0e9d80a9c154d45652a9e16..3d68f627aad52f0432da8682846d3501fc4bb5ba 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -56,7 +56,9 @@ PROVIDERS = [ # labels it already has in front of it. {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", "url": "https://api.anthropic.com/v1/messages", - "model": "claude-haiku-4-5"}, + # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, + # and a deployment whose side rungs are out of credit can raise the tier without a release. + "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, ] ANTHROPIC_VERSION = "2023-06-01" TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) @@ -277,13 +279,23 @@ MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engi MAX_PROMPT_CHARS = 2000 -def flow_providers(): - """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.""" - pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() +def flow_providers(pin=None): + """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. + + ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and + the draft door used to read `prompt` off the body and nothing else — so the key the client sent + was accepted and dropped, and the picker was a control over nothing. + ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the + ladder stopped offering must not turn every later draft into an error; the caller is told which + rung actually answered, which is the honest half. + """ by_name = {p["name"]: p for p in PROVIDERS} - order = [pin] if pin else list(FLOW_PROVIDER_ORDER) - return [by_name[n] for n in order + live = [n for n in FLOW_PROVIDER_ORDER if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] + wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() + if wanted and wanted in live: + return [by_name[wanted]] + return [by_name[n] for n in live] def flow_schema(kinds, trigger_keys, table_keys): @@ -421,7 +433,7 @@ def _flow_from_tool_call(obj): def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, - st=None, user=""): + st=None, user="", model=None): """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. Exactly one of `draft` and `refusal_sentence` is truthy — the same contract @@ -456,7 +468,7 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou draft, refusal = _flow_from_tool_call(chat(messages, tools)) return draft, refusal, "injected" - provs = flow_providers() + provs = flow_providers(model) if not provs: # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was # never built [[flag-shipped-without-its-writer]]. @@ -464,30 +476,53 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou "be drafted from a description yet"), None tmo = float(timeout or TIMEOUT_SECONDS) problems = [] + import providers as _prov for p in provs: - if p["shape"] != "openai": - # Anthropic's tool transport differs; it stays on the ladder for `decide()` and is - # SKIPPED here rather than half-supported. Named in `problems` so it is not invisible. - problems.append(f"{p['name']}: tool calls are not wired for this shape") - continue + # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read + # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, + # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the + # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* + # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: + # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter + # 402), so without this branch the drafter cannot answer at all. + # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this + # ladder runs. The parameter is the caller's to send, which is why the wire takes it. try: - r = requests.post(p["url"], timeout=tmo, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1500}) + if p["shape"] == "anthropic": + req = _prov.anthropic_request( + model=p["model"], key=os.environ[p["env"]].strip(), system=None, + messages=messages, tools=tools, max_tokens=1500, tool_choice="required") + r = requests.post(req["url"], timeout=tmo, + headers=req["headers"], json=req["json"]) + else: + r = requests.post(p["url"], timeout=tmo, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1500}) except Exception as e: # noqa: BLE001 problems.append(f"{p['name']}: {type(e).__name__}") continue if r.status_code != 200: - problems.append(f"{p['name']}: HTTP {r.status_code}") + # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off + # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and + # the memo makes the next turn SKIP the empty account instead of paying for it again. + if _prov.is_credit_failure(r.status_code, r.text): + _prov.mark_no_credit(p["name"]) + problems.append(_prov.refusal_sentence(p["name"], r.status_code, r.text)) continue try: body = r.json() - calls = (((body.get("choices") or [{}])[0].get("message") or {}) - .get("tool_calls") or []) - args = json.loads(calls[0]["function"]["arguments"]) if calls else None + if p["shape"] == "anthropic": + _text, args, _refused = _prov.anthropic_read(body) + if _refused: + problems.append(f"{p['name']}: {_refused}") + continue + else: + calls = (((body.get("choices") or [{}])[0].get("message") or {}) + .get("tool_calls") or []) + args = json.loads(calls[0]["function"]["arguments"]) if calls else None except Exception as e: # noqa: BLE001 problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") continue diff --git a/api/automation_engine.py b/api/automation_engine.py index 5d60218280cb2c33520bb31163c4e64ea5f6adec..a696848fecf36f2669c5aa3d3edbe5295ae85403 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -9219,6 +9219,36 @@ ACTION_CATALOG = [ "ready": True, "detail": "Assemble this month's statements and park them for review. Nothing is sent until " "somebody opens the batch and clicks Send"}, + # ⭐⭐ WAVE 36 · W36-T39 — D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an + # `ACTION_CATALOG` row with `menu: false` — one line, reusing the door R18 built this same wave + # to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."* + # + # ⛔ THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows` + # emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and + # that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The + # builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls + # back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN. + # + # ⚠ `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" — a promise — + # and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment + # anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker, + # still resolvable as a caption, still valid. + # ⛔ AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation + # canvas is not where one is created, which is why `patch_automation` already refuses a + # `field:` id with "change its prompt, model or schedule on the column itself". + # ⭐⭐ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row` + # emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and + # that kind was in no catalog either — the same raw token on the same screen as D-277, one row + # down. Two known instances were enough to justify the check; the check then produced a third + # nobody had booked, which is the difference between a gate and a regression test. + {"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected", + "ready": True, "menu": False, + "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not " + "here"}, + {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected", + "ready": True, "menu": False, + "detail": "Fill an AI column for the records this flow walks. Configured on the column, not " + "here"}, {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, "detail": "Needs the Slack connector"}, # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── @@ -9306,6 +9336,33 @@ def _tenant_may_use(kind, rt): return True if gate is None else bool(rt is not None and gate(rt)) +def catalog_kinds(): + """Every action kind this module knows about — the LABEL vocabulary. + + ⭐ D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the + catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog + does not carry is a token on somebody's screen. This is the set that must cover every kind any + server path can put into a `flow.actions` entry, whether or not a person may add it. + """ + return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG) + + +def configurable_kinds(): + """The kinds a PERSON may add from the picker, and must therefore be able to configure. + + ⭐⭐ W36-T39 — THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is + the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is + ready and `menu: False` (D-277) — all six are real, runnable, captioned, and unofferable. What + a client must be able to CONFIGURE is exactly what a person can ADD. + + ⛔ DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list + would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile + were already one too many. + """ + return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG + if row.get("ready") and row.get("menu", True) is not False) + + def action_catalog(rt=None): """The catalog as the wire carries it — a copy, because a caller that mutated the module constant would change every later reader's answer. @@ -10831,6 +10888,32 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print "and nothing was sent.") counts["webBlocked"] += 1 continue + elif kind == "odoo_sync": + # Same refusal, same reason as the arm below: a catalog row with no arm is walked, + # counted, and reports success having done nothing. The connector owns this sync. + if "odoo_sync_not_run_here" not in web_notes: + web_notes.append("odoo_sync_not_run_here") + log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own " + "schedule, not from this canvas. Nothing was done") + counts["webBlocked"] += 1 + continue + elif kind == "ai_enrich": + # ⛔⛔ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE + # STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is + # walked, COUNTED, reports the run `ok` and writes nothing — a step somebody + # configured that succeeds at doing nothing. Adding the row (D-277) without this + # arm would have opened exactly that window. + # ⚠ AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by + # `ai_enrich`, driven from the column's own editor; the synthetic agent row this + # kind appears in is DERIVED from that column and is never stored, so nothing + # reaches here through the ordinary path. If something ever does, it must say so + # rather than report success. + if "ai_enrich_not_run_here" not in web_notes: + web_notes.append("ai_enrich_not_run_here") + log("[aios-auto] ai_enrich: an AI column is filled from the column's own " + "editor, not from this canvas. Nothing was done") + counts["webBlocked"] += 1 + continue elif kind == "ai_agent": # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. # diff --git a/api/connectors_tt.py b/api/connectors_tt.py index 60d96945a35f7fd8ca79443bf70cdec87e0a4c53..0603f4f23f7075252fdcef06ce19371ac1620523 100644 --- a/api/connectors_tt.py +++ b/api/connectors_tt.py @@ -1,568 +1,568 @@ -"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). - -Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an -automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file -exists at all rather than another thousand lines inside the engine. - -⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 -post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read -live from `GET /datasets/{id}/metadata` for **$0.00** and written down in -`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at -close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a -name below reads through a candidate list it is because the vendor has two names for one fact -(`biography`/`signature`, `region`/`country`), never because the name is uncertain. - -⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — -public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The -vendor key is a key to a SUPPLIER. - -⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE -OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and -`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not -Instagram's — and re-implementing them here would be a second copy of the deferral handling, the -truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. -Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from -`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. -⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because -the sentence above is the kind that quietly stops being true. - -⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** - 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared - envelope keys (`timestamp`, `input`) that no metadata call mentions; - 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` - and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one - TikTok claim that matters most (`play_count`) is exactly a declaration. -""" -from __future__ import annotations - -import automation_engine as engine -# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an -# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were -# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever -# read them off this module (the engine imports them from the transport itself), so they were four -# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. -from connectors_bd import ( - _bd_first_url, - _bd_flag, - _bd_list, - _bd_source_payload, - _first, - _ig_int, - bd_scrape, -) - -# --------------------------------------------------------------------------------------------- -# THE DATASETS -# --------------------------------------------------------------------------------------------- -# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET -# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a -# full field list, and the two DISCOVERY halves answer **404 for our key**: -# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). -# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's -# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. -TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records -TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields -TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields - -#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` -#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. -TT_POST_TYPE_VIDEO = "video" -TT_POST_TYPE_CONTENT = "content" - -#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept -#: beside the dataset ids because it is the same class of vendor fact. -#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical -#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own -#: batch calls, and the gate asserts the two agree rather than trusting that they do. -TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" - - -def tt_profile_url(handle): - """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" - h = str(handle or "").strip().lstrip("@") - return TT_PROFILE_URL.format(handle=h) if h else "" - - -# --------------------------------------------------------------------------------------------- -# THE FIELD MAPS -# --------------------------------------------------------------------------------------------- -# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared -# in `automation_engine`. The rules they all obey, stated once: -# -# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, -# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` -# returns `None` (never 0) when nothing matches, which is what makes that possible. -# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` -# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) -# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the -# vendor's side is preserved rather than silently discarded while our column model catches up. -# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` -# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they -# ride in `source_payload` where they make no claim. - - -def _tt_str(node, *names): - """The first non-empty string among `names`, or None when the vendor sent nothing. - - ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — - an empty string written into a cell claims "we looked and it is empty". - """ - v = _first(node, *names) - if v is None: - return None - s = str(v).strip() - return s or None - - -def _tt_pct(node, *names): - """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. - - ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to - the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — - measured on the Instagram side, and TikTok sends the same shape on all three of its rates. - """ - v = _first(node, *names) - if v is None: - return None - out = engine._pct100(v) - return out or None - - -def _tt_day(node, *names): - """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" - v = _first(node, *names) - if v is None: - return None - return engine._day(v) or None - - -def _drop_blanks(row): - """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" - return {k: v for k, v in row.items() if v is not None and v != ""} - - -def normalize_profile(node, handle=""): - """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. - - ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: - * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` - populated on 85% of rows and they carry the same text; - * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented - two-letter-ISO description, `country` has no description at all. - ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its - `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the - only count of its kind the dataset offers. - """ - node = node if isinstance(node, dict) else {} - account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "handle": account, - "full_name": _tt_str(node, "nickname"), - "tt_id": _tt_str(node, "id"), - "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), - "bio": _tt_str(node, "biography", "signature"), - "external_url": _bd_first_url(_first(node, "bio_link")), - "verified": _bd_flag(node, "is_verified"), - "is_private": _bd_flag(node, "is_private"), - # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* - # on their own description. It is the closest thing TikTok has to Instagram's - # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so - # the approximation only ever fills a cell the vendor actually answered. - "is_business": _bd_flag(node, "is_commerce_user"), - "followers": _ig_int(_first(node, "followers")), - "following": _ig_int(_first(node, "following")), - "posts_count": _ig_int(_first(node, "videos_count")), - # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no - # nulls). It is not a post-level number and it is not our `likes` column, which is why it - # is stored under a different name. - "likes_received": _ig_int(_first(node, "likes")), - "avg_engagement": _tt_pct(node, "awg_engagement_rate"), - "like_engagement": _tt_pct(node, "like_engagement_rate"), - "comment_engagement": _tt_pct(node, "comment_engagement_rate"), - "country_code": _tt_str(node, "region", "country"), - "region": _tt_str(node, "region"), - "predicted_lang": _tt_str(node, "predicted_lang"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT - # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, - # which is why the append law dates a snapshot by when WE read it. - "account_created_at": _tt_day(node, "create_time"), - "source_payload": _bd_source_payload(node), - }) - - -def tt_post_type(node): - """A TikTok post row → one of OUR three type options, or None. - - ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / - `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the - vendor's token would fail `_clean_field`'s option check on the way in and would put an - untranslated API word in front of a user on the way out. - - So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the - row carries more than one `carousel_images` entry, which is what a carousel IS on either - network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the - token cannot express it, so inferring `carousel` from the word would be inventing a fact. - ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and - that is exactly what would make the wrong default invisible). - """ - raw = str((node or {}).get("post_type") or "").strip().lower() - if raw == TT_POST_TYPE_VIDEO: - return "video" - if raw != TT_POST_TYPE_CONTENT: - return None - images = (node or {}).get("carousel_images") - return "carousel" if isinstance(images, list) and len(images) > 1 else "image" - - -def normalize_post(node): - """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. - - ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and - Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same - fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and - `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would - manufacture a second measurement that a rollup could average or double-count, which is a worse - outcome than the missing column it would paper over. - - ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and - the probe measured them as the same shape. That equality is what lets the comments→posts link - join with NO normaliser, which the Instagram side never had. - ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). - ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and - countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares - no equivalent, so `paid_partnership`/`partner` have no column on this family at all. - """ - node = node if isinstance(node, dict) else {} - shortcode = _tt_str(node, "shortcode", "post_id") - if not shortcode: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "shortcode": shortcode, - # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. - # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the - # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so - # one name means one thing across both corpora. Reading the display name silently broke the - # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched - # NOTHING, so a person could not filter posts by creator and a rollup would count zero. - # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a - # different fact, and filling a join key with it is worse than leaving it blank — a blank - # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. - # The URL is the honest second source: it carries the handle by construction. - "influencer_key": (_tt_str(node, "account_id") - or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), - "posted_at": _tt_day(node, "create_time"), - "type": tt_post_type(node), - "caption": _tt_str(node, "description"), - "url": _tt_str(node, "url"), - "hashtags": _bd_list(node, "hashtags"), - "tagged_location": _tt_str(node, "commerce_info"), - "views": _ig_int(_first(node, "play_count")), - "likes": _ig_int(_first(node, "digg_count")), - "comments": _ig_int(_first(node, "comment_count")), - "shares": _ig_int(_first(node, "num_share_count")), - "saves": _ig_int(_first(node, "collect_count")), - "video_duration": _ig_int(_first(node, "video_duration")), - "source_payload": _bd_source_payload(node), - }) - - -def normalize_comment(node): - """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. - - ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR - `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in - `source_payload`. Reading the array's length instead would be a second, disagreeing answer to - a question the vendor already answers — and it would disagree, because a page of replies is not - all of them. - ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is - text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor - that changes its mind cannot change ours. - ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged - PII) stay in `source_payload` and are promoted to no column, which is the same posture the - Instagram comment schema takes for the same D-24 reason. - """ - node = node if isinstance(node, dict) else {} - comment_key = _tt_str(node, "comment_id") - if not comment_key: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "comment_key": comment_key, - "shortcode": _tt_str(node, "post_id"), - # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the - # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); - # primary first, so a row carrying the rich form is not silently served the plain one. - # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. - "text": _tt_str(node, "comment_text", "comment_text_only"), - "commented_at": _tt_day(node, "date_created"), - "likes": _ig_int(_first(node, "num_likes")), - "replies": _ig_int(_first(node, "num_replies")), - "source_payload": _bd_source_payload(node), - }) - - -#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them -#: rather than naming three functions, and so adding a fourth dataset is one entry. -TT_NORMALIZERS = { - "tt_profile": normalize_profile, - "tt_post_metrics": normalize_post, - "tt_comments": normalize_comment, -} - - -# --------------------------------------------------------------------------------------------- -# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) -# --------------------------------------------------------------------------------------------- - -def tt_handle(url): - """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. - - Deliberately permissive about the input and strict about the output, because the two callers - hand it different things: an automation stores whatever a person typed in the profile column - (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean - `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot - resolve to two different handles. - """ - s = str(url or "").strip() - if not s: - return "" - if "tiktok.com" in s.lower(): - # Everything after the first `@`, up to the next path segment or query. - tail = s.split("@", 1)[1] if "@" in s else "" - s = tail.split("/")[0].split("?")[0].split("#")[0] - s = s.strip().lstrip("@").strip() - # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. - return s if s and all(c.isalnum() or c in "._" for c in s) else "" - - -def tt_post_urls(node, limit=0): - """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. - - ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row - we have already bought, so the posts read is a scrape of links we hold, never a search for them. - The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, - so a design that reached for either would not merely be dearer, it would not work. - - ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an - array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe - read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's - `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** - (measured below), so the reader built against the quoted sentence found nothing, forever, in - silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration - of X"* are different claims, and prose cannot be told apart by a reader downstream — which is - why the correction names the method, not just the value. - - ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, - and preferring whichever happened to be longer is how one creator's window silently differs - from another's. - ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, - validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is - never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, - not a number this function may invent. - """ - raw = (node or {}).get("top_videos") - out = [] - for item in raw if isinstance(raw, list) else []: - # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. - # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a - # live handle returns **19 DICTS**, keyed - # `video_url · video_id · playcount · diggcount · commentcount · share_count · - # favorites_count · create_date · cover_image`. - # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and - # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says - # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field - # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's - # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was - # the first. [[reachable-is-not-the-same-as-built]] - # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY - # list on every real profile — post capture could not have worked for anybody, and the - # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture - # written from a schema tests the schema. - # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept - # because it costs nothing and is what a future corpus revision would most likely use. The - # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents - # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as - # before, rather than to a URL built out of a guess. - # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring - # whichever array happened to be longer is how one creator's window silently differs from - # another's, and that reasoning is unchanged by this correction. - if isinstance(item, dict): - u = str(item.get("video_url") or item.get("url") or "").strip() - else: - u = str(item or "").strip() - if u.lower().startswith("http") and u not in out: - out.append(u) - return out[:limit] if limit and limit > 0 else out - - -def pull_posts_tt(post_urls, log=print, deferred=None): - """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. - - ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side - measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still - running at 67 minutes. Nothing here loops per URL. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] posts: {note}") - return [], note - out = [r for r in (normalize_post(n) for n in rows) if r] - return out, "" - - -def pull_comments_tt(post_urls, log=print, deferred=None): - """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. - - ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on - both networks: a comments scrape ingests identifiable third parties who never entered anybody's - list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes - none of them to a column; this function adds no new exposure, it just has to be asked for. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] comments: {note}") - return [], note - out = [r for r in (normalize_comment(n) for n in rows) if r] - return out, "" - - -#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. -#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral -#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's -#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse -#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. -TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) - - -def _media_deferrals(deferred): - """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. - - ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the - handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one - kind here, so the id already carries everything a mapper choice depends on — and importing - Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). - """ - out = [] - for d in deferred or []: - if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: - out.append(dict(d)) - return out - - -def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, - max_posts=0, post_metrics=False, comment_metrics=False): - """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. - - `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, - so the engine's enrich branch treats every network identically and no caller learns a new - shape. - - ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's - do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — - no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` - then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction - rather than by a rule: their input IS a post permalink, so asking for comments with post capture - off is a request with no subject, and it returns none instead of quietly buying posts nobody - asked for. - - ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than - pessimistic. The Instagram contract reads `ok` only when identity AND media both landed - (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower - count and no posts must not paint green over a posts table that did not grow"*). So: posts not - ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → - `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", - never "did the function finish". - - ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the - paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, - with its own note explaining that a multi-provider chain is a promise something walks it and - that nothing walks Instagram's second name today either. So a refusal here is final, and it - says so instead of implying a retry somewhere. - """ - handle = tt_handle(url) - if not handle: - return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", - "note": f"{url!r} is not a TikTok profile URL or handle"} - - # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from - # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not - # happen; a miss falls through to the single-URL call below. - cached = prefetch.get(handle) if isinstance(prefetch, dict) else None - _deferred = [] - if isinstance(cached, dict) and cached: - rows, note = [cached], "" - else: - rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) - - node = rows[0] if rows else {} - profile = normalize_profile(node, handle) if node else {} - # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` - # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and - # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly - # this reason, and answering "0 followers" instead is the failure it exists to prevent. - unreadable = profile.get("followers") is None and profile.get("following") is None - if note or unreadable: - # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still - # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the - # same record. That was live on the Instagram profile path until 2026-08-09 — measured on - # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being - # reintroduced here by omission. - if isinstance(pending_profile, list): - for d in _deferred: - pending_profile.append({**d, "kind": "profile", "influencer": handle}) - why = note or ("the scrape answered, but no follower/following counts were readable in it " - "(the field names may have moved - see tiktok-capture.md)") - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "deferredProfile": [d.get("snapshotId") for d in _deferred], - "note": why} - - # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ - if not post_metrics: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; post capture is off for this step"} - urls = tt_post_urls(node, limit=max_posts) - if not urls: - # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and - # saying so is what stops the next run paying to be told the same thing. - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; this account's row carried no post links"} - posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) - comments, c_note = ([], "") - if comment_metrics and posts: - # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — - # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused - # is not silently asked about again one rung later. - comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], - log=log, deferred=_deferred) - # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is - # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to - # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE - # deferral is impossible — the profile branch above returns `blocked` on any note — so **every - # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose - # collector writes preset profile cells onto somebody's record from post rows. The engine keeps - # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). - # ⇒ So this returns them under their OWN key, filtered by dataset identity - # (`_media_deferrals`), and the engine files them in the metric queue with the handle it - # already holds. Returning rather than appending also keeps the queue's vocabulary out of a - # connector: this module knows which CORPUS deferred, never what the engine calls it. - # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that - # matters most — that is exactly the run where the vendor took too long, so a caller reading - # the ids only from the success path would lose every batch it actually paid for. - deferred_media = _media_deferrals(_deferred) - if not posts: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", "deferredMedia": deferred_media, - "note": p_note or note or "profile read; the post source returned nothing"} - return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, - "via": "brightdata", "deferredMedia": deferred_media, - "note": c_note or note or ""} +"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). + +Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an +automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file +exists at all rather than another thousand lines inside the engine. + +⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 +post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read +live from `GET /datasets/{id}/metadata` for **$0.00** and written down in +`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a +name below reads through a candidate list it is because the vendor has two names for one fact +(`biography`/`signature`, `region`/`country`), never because the name is uncertain. + +⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — +public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The +vendor key is a key to a SUPPLIER. + +⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE +OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and +`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not +Instagram's — and re-implementing them here would be a second copy of the deferral handling, the +truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. +Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from +`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. +⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because +the sentence above is the kind that quietly stops being true. + +⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** + 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared + envelope keys (`timestamp`, `input`) that no metadata call mentions; + 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` + and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one + TikTok claim that matters most (`play_count`) is exactly a declaration. +""" +from __future__ import annotations + +import automation_engine as engine +# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an +# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were +# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever +# read them off this module (the engine imports them from the transport itself), so they were four +# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. +from connectors_bd import ( + _bd_first_url, + _bd_flag, + _bd_list, + _bd_source_payload, + _first, + _ig_int, + bd_scrape, +) + +# --------------------------------------------------------------------------------------------- +# THE DATASETS +# --------------------------------------------------------------------------------------------- +# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET +# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a +# full field list, and the two DISCOVERY halves answer **404 for our key**: +# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). +# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's +# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. +TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records +TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields +TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields + +#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` +#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. +TT_POST_TYPE_VIDEO = "video" +TT_POST_TYPE_CONTENT = "content" + +#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept +#: beside the dataset ids because it is the same class of vendor fact. +#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical +#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own +#: batch calls, and the gate asserts the two agree rather than trusting that they do. +TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" + + +def tt_profile_url(handle): + """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" + h = str(handle or "").strip().lstrip("@") + return TT_PROFILE_URL.format(handle=h) if h else "" + + +# --------------------------------------------------------------------------------------------- +# THE FIELD MAPS +# --------------------------------------------------------------------------------------------- +# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared +# in `automation_engine`. The rules they all obey, stated once: +# +# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, +# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` +# returns `None` (never 0) when nothing matches, which is what makes that possible. +# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` +# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) +# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the +# vendor's side is preserved rather than silently discarded while our column model catches up. +# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` +# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they +# ride in `source_payload` where they make no claim. + + +def _tt_str(node, *names): + """The first non-empty string among `names`, or None when the vendor sent nothing. + + ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — + an empty string written into a cell claims "we looked and it is empty". + """ + v = _first(node, *names) + if v is None: + return None + s = str(v).strip() + return s or None + + +def _tt_pct(node, *names): + """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. + + ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to + the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — + measured on the Instagram side, and TikTok sends the same shape on all three of its rates. + """ + v = _first(node, *names) + if v is None: + return None + out = engine._pct100(v) + return out or None + + +def _tt_day(node, *names): + """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" + v = _first(node, *names) + if v is None: + return None + return engine._day(v) or None + + +def _drop_blanks(row): + """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" + return {k: v for k, v in row.items() if v is not None and v != ""} + + +def normalize_profile(node, handle=""): + """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. + + ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: + * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` + populated on 85% of rows and they carry the same text; + * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented + two-letter-ISO description, `country` has no description at all. + ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its + `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the + only count of its kind the dataset offers. + """ + node = node if isinstance(node, dict) else {} + account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "handle": account, + "full_name": _tt_str(node, "nickname"), + "tt_id": _tt_str(node, "id"), + "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), + "bio": _tt_str(node, "biography", "signature"), + "external_url": _bd_first_url(_first(node, "bio_link")), + "verified": _bd_flag(node, "is_verified"), + "is_private": _bd_flag(node, "is_private"), + # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* + # on their own description. It is the closest thing TikTok has to Instagram's + # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so + # the approximation only ever fills a cell the vendor actually answered. + "is_business": _bd_flag(node, "is_commerce_user"), + "followers": _ig_int(_first(node, "followers")), + "following": _ig_int(_first(node, "following")), + "posts_count": _ig_int(_first(node, "videos_count")), + # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no + # nulls). It is not a post-level number and it is not our `likes` column, which is why it + # is stored under a different name. + "likes_received": _ig_int(_first(node, "likes")), + "avg_engagement": _tt_pct(node, "awg_engagement_rate"), + "like_engagement": _tt_pct(node, "like_engagement_rate"), + "comment_engagement": _tt_pct(node, "comment_engagement_rate"), + "country_code": _tt_str(node, "region", "country"), + "region": _tt_str(node, "region"), + "predicted_lang": _tt_str(node, "predicted_lang"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT + # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, + # which is why the append law dates a snapshot by when WE read it. + "account_created_at": _tt_day(node, "create_time"), + "source_payload": _bd_source_payload(node), + }) + + +def tt_post_type(node): + """A TikTok post row → one of OUR three type options, or None. + + ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / + `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the + vendor's token would fail `_clean_field`'s option check on the way in and would put an + untranslated API word in front of a user on the way out. + + So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the + row carries more than one `carousel_images` entry, which is what a carousel IS on either + network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the + token cannot express it, so inferring `carousel` from the word would be inventing a fact. + ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and + that is exactly what would make the wrong default invisible). + """ + raw = str((node or {}).get("post_type") or "").strip().lower() + if raw == TT_POST_TYPE_VIDEO: + return "video" + if raw != TT_POST_TYPE_CONTENT: + return None + images = (node or {}).get("carousel_images") + return "carousel" if isinstance(images, list) and len(images) > 1 else "image" + + +def normalize_post(node): + """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. + + ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and + Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same + fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and + `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would + manufacture a second measurement that a rollup could average or double-count, which is a worse + outcome than the missing column it would paper over. + + ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and + the probe measured them as the same shape. That equality is what lets the comments→posts link + join with NO normaliser, which the Instagram side never had. + ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). + ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and + countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares + no equivalent, so `paid_partnership`/`partner` have no column on this family at all. + """ + node = node if isinstance(node, dict) else {} + shortcode = _tt_str(node, "shortcode", "post_id") + if not shortcode: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "shortcode": shortcode, + # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. + # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the + # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so + # one name means one thing across both corpora. Reading the display name silently broke the + # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched + # NOTHING, so a person could not filter posts by creator and a rollup would count zero. + # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a + # different fact, and filling a join key with it is worse than leaving it blank — a blank + # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. + # The URL is the honest second source: it carries the handle by construction. + "influencer_key": (_tt_str(node, "account_id") + or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), + "posted_at": _tt_day(node, "create_time"), + "type": tt_post_type(node), + "caption": _tt_str(node, "description"), + "url": _tt_str(node, "url"), + "hashtags": _bd_list(node, "hashtags"), + "tagged_location": _tt_str(node, "commerce_info"), + "views": _ig_int(_first(node, "play_count")), + "likes": _ig_int(_first(node, "digg_count")), + "comments": _ig_int(_first(node, "comment_count")), + "shares": _ig_int(_first(node, "num_share_count")), + "saves": _ig_int(_first(node, "collect_count")), + "video_duration": _ig_int(_first(node, "video_duration")), + "source_payload": _bd_source_payload(node), + }) + + +def normalize_comment(node): + """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. + + ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR + `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in + `source_payload`. Reading the array's length instead would be a second, disagreeing answer to + a question the vendor already answers — and it would disagree, because a page of replies is not + all of them. + ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is + text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor + that changes its mind cannot change ours. + ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged + PII) stay in `source_payload` and are promoted to no column, which is the same posture the + Instagram comment schema takes for the same D-24 reason. + """ + node = node if isinstance(node, dict) else {} + comment_key = _tt_str(node, "comment_id") + if not comment_key: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "comment_key": comment_key, + "shortcode": _tt_str(node, "post_id"), + # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the + # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); + # primary first, so a row carrying the rich form is not silently served the plain one. + # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. + "text": _tt_str(node, "comment_text", "comment_text_only"), + "commented_at": _tt_day(node, "date_created"), + "likes": _ig_int(_first(node, "num_likes")), + "replies": _ig_int(_first(node, "num_replies")), + "source_payload": _bd_source_payload(node), + }) + + +#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them +#: rather than naming three functions, and so adding a fourth dataset is one entry. +TT_NORMALIZERS = { + "tt_profile": normalize_profile, + "tt_post_metrics": normalize_post, + "tt_comments": normalize_comment, +} + + +# --------------------------------------------------------------------------------------------- +# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) +# --------------------------------------------------------------------------------------------- + +def tt_handle(url): + """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. + + Deliberately permissive about the input and strict about the output, because the two callers + hand it different things: an automation stores whatever a person typed in the profile column + (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean + `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot + resolve to two different handles. + """ + s = str(url or "").strip() + if not s: + return "" + if "tiktok.com" in s.lower(): + # Everything after the first `@`, up to the next path segment or query. + tail = s.split("@", 1)[1] if "@" in s else "" + s = tail.split("/")[0].split("?")[0].split("#")[0] + s = s.strip().lstrip("@").strip() + # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. + return s if s and all(c.isalnum() or c in "._" for c in s) else "" + + +def tt_post_urls(node, limit=0): + """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. + + ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row + we have already bought, so the posts read is a scrape of links we hold, never a search for them. + The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, + so a design that reached for either would not merely be dearer, it would not work. + + ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an + array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe + read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's + `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** + (measured below), so the reader built against the quoted sentence found nothing, forever, in + silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration + of X"* are different claims, and prose cannot be told apart by a reader downstream — which is + why the correction names the method, not just the value. + + ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, + and preferring whichever happened to be longer is how one creator's window silently differs + from another's. + ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, + validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is + never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, + not a number this function may invent. + """ + raw = (node or {}).get("top_videos") + out = [] + for item in raw if isinstance(raw, list) else []: + # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. + # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a + # live handle returns **19 DICTS**, keyed + # `video_url · video_id · playcount · diggcount · commentcount · share_count · + # favorites_count · create_date · cover_image`. + # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and + # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says + # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field + # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's + # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was + # the first. [[reachable-is-not-the-same-as-built]] + # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY + # list on every real profile — post capture could not have worked for anybody, and the + # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture + # written from a schema tests the schema. + # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept + # because it costs nothing and is what a future corpus revision would most likely use. The + # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents + # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as + # before, rather than to a URL built out of a guess. + # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring + # whichever array happened to be longer is how one creator's window silently differs from + # another's, and that reasoning is unchanged by this correction. + if isinstance(item, dict): + u = str(item.get("video_url") or item.get("url") or "").strip() + else: + u = str(item or "").strip() + if u.lower().startswith("http") and u not in out: + out.append(u) + return out[:limit] if limit and limit > 0 else out + + +def pull_posts_tt(post_urls, log=print, deferred=None): + """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. + + ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side + measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still + running at 67 minutes. Nothing here loops per URL. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] posts: {note}") + return [], note + out = [r for r in (normalize_post(n) for n in rows) if r] + return out, "" + + +def pull_comments_tt(post_urls, log=print, deferred=None): + """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. + + ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on + both networks: a comments scrape ingests identifiable third parties who never entered anybody's + list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes + none of them to a column; this function adds no new exposure, it just has to be asked for. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] comments: {note}") + return [], note + out = [r for r in (normalize_comment(n) for n in rows) if r] + return out, "" + + +#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. +#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral +#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's +#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse +#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. +TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) + + +def _media_deferrals(deferred): + """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. + + ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the + handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one + kind here, so the id already carries everything a mapper choice depends on — and importing + Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). + """ + out = [] + for d in deferred or []: + if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: + out.append(dict(d)) + return out + + +def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, + max_posts=0, post_metrics=False, comment_metrics=False): + """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. + + `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, + so the engine's enrich branch treats every network identically and no caller learns a new + shape. + + ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's + do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — + no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` + then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction + rather than by a rule: their input IS a post permalink, so asking for comments with post capture + off is a request with no subject, and it returns none instead of quietly buying posts nobody + asked for. + + ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than + pessimistic. The Instagram contract reads `ok` only when identity AND media both landed + (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower + count and no posts must not paint green over a posts table that did not grow"*). So: posts not + ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → + `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", + never "did the function finish". + + ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the + paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, + with its own note explaining that a multi-provider chain is a promise something walks it and + that nothing walks Instagram's second name today either. So a refusal here is final, and it + says so instead of implying a retry somewhere. + """ + handle = tt_handle(url) + if not handle: + return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", + "note": f"{url!r} is not a TikTok profile URL or handle"} + + # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from + # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not + # happen; a miss falls through to the single-URL call below. + cached = prefetch.get(handle) if isinstance(prefetch, dict) else None + _deferred = [] + if isinstance(cached, dict) and cached: + rows, note = [cached], "" + else: + rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) + + node = rows[0] if rows else {} + profile = normalize_profile(node, handle) if node else {} + # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` + # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and + # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly + # this reason, and answering "0 followers" instead is the failure it exists to prevent. + unreadable = profile.get("followers") is None and profile.get("following") is None + if note or unreadable: + # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still + # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the + # same record. That was live on the Instagram profile path until 2026-08-09 — measured on + # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being + # reintroduced here by omission. + if isinstance(pending_profile, list): + for d in _deferred: + pending_profile.append({**d, "kind": "profile", "influencer": handle}) + why = note or ("the scrape answered, but no follower/following counts were readable in it " + "(the field names may have moved - see tiktok-capture.md)") + return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", + "deferredProfile": [d.get("snapshotId") for d in _deferred], + "note": why} + + # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ + if not post_metrics: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; post capture is off for this step"} + urls = tt_post_urls(node, limit=max_posts) + if not urls: + # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and + # saying so is what stops the next run paying to be told the same thing. + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; this account's row carried no post links"} + posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) + comments, c_note = ([], "") + if comment_metrics and posts: + # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — + # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused + # is not silently asked about again one rung later. + comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], + log=log, deferred=_deferred) + # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is + # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to + # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE + # deferral is impossible — the profile branch above returns `blocked` on any note — so **every + # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose + # collector writes preset profile cells onto somebody's record from post rows. The engine keeps + # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). + # ⇒ So this returns them under their OWN key, filtered by dataset identity + # (`_media_deferrals`), and the engine files them in the metric queue with the handle it + # already holds. Returning rather than appending also keeps the queue's vocabulary out of a + # connector: this module knows which CORPUS deferred, never what the engine calls it. + # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that + # matters most — that is exactly the run where the vendor took too long, so a caller reading + # the ids only from the success path would lose every batch it actually paid for. + deferred_media = _media_deferrals(_deferred) + if not posts: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", "deferredMedia": deferred_media, + "note": p_note or note or "profile read; the post source returned nothing"} + return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, + "via": "brightdata", "deferredMedia": deferred_media, + "note": c_note or note or ""} diff --git a/api/main.py b/api/main.py index 175e0173f243e89cf5f436e877e45e6b6317562b..239967d71274c306d56fe8e0bf614a7dff8eb76c 100644 --- a/api/main.py +++ b/api/main.py @@ -85,6 +85,8 @@ import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack d import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) +import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) +import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) from core import grid_events # noqa: E402 # D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a @@ -376,6 +378,23 @@ app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one # POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does # not widen anything a tenant admin can reach — `verify_api` proves that by having one try. app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane +# ⭐⭐ WAVE 36 (R8 / C4) — THE AGENT HARNESS FILE STORE, mounted in the SAME change that created +# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the +# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in +# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107). +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green. +# ⛔ ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES — and that is safe +# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot +# match `/agents/x/harness`. The two routers share a prefix and no route. +app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent harness files +# ⭐⭐ WAVE 36 (R3 / R10 / C3) — THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that +# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in +# `app.openapi()["paths"]` AND calls them, and its NC comments this line out. +# ⛔ ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool. +# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header +# says why that is not a style choice. +app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 3659f76145d0bfdeec4ec26fc75c2d8d6878faa0..9a2f2aba3fceb4b2d70b7827afec3b38d4f34928 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -1,1824 +1,1824 @@ -"""odoo_relational.py — Odoo entities as LOCKED relational databases. - -Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset -Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually -replace. - -⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the -owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports -tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the -partners who owed money — so most Odoo ids were simply absent, and the missing rows were the -reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: - - ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a - posted customer document - ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code - ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund - ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order - -⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's -own `plan()` refused above it — but the cap was never a property of the store (see the measured -banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). -The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until -wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory -budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). -The four tables together are -17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split -is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); -they are answered by the read-through rollup, which never copies a row. - -⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts -GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables -dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and -carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner -holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. -Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you -filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps -excluding, because `modules/ar` is its oracle and an oracle answers ONE question. - -⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over -ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate -**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside -`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the -`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the -wider link would count 31,418 documents and call them open invoices, so those two rollups carry -the oracle's predicate as an explicit `payment_state` condition pair. -""" -import datetime as _dt - -#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: -#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. -RI_SLUGS = ("", "royal-imports") - -INVOICES_KEY = "ut_odoo_invoices" -CUSTOMERS_KEY = "ut_odoo_customers" -ORDERS_KEY = "ut_odoo_orders" -PRODUCTS_KEY = "ut_odoo_products" -#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, -#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to -#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). -AGENTS_KEY = "ut_odoo_agents" -ACCOUNTS_KEY = "ut_odoo_accounts" -BILLS_KEY = "ut_odoo_bills" -VENDORS_KEY = "ut_odoo_vendors" - -#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. -#: -#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it -#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced -#: rather than left standing beside its own contradiction. The obstacle was never the number of -#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED -#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) -#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 -#: settles what that means: *"there is no cap in how many data from the API source … can be pulled -#: into the app"*, so the answer is a different residency, never a bigger ceiling. -#: -#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the -#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this -#: database stores no rows HERE", which is a different statement from `None` ("connected and -#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator -#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work -#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with -#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working -#: grid; that is the whole shape of the conversion. -ORDER_LINES_KEY = "ut_odoo_order_lines" -GL_LINES_KEY = "ut_odoo_gl_lines" - -#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). -JOIN_KEY = "partner_id" -#: The product-grain equivalent. -PRODUCT_JOIN_KEY = "product_id" -#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but -#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the -#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. -AGENT_JOIN_KEY = "agent_id" -#: A vendor is also a `res.partner`; same reasoning, its own column. -VENDOR_JOIN_KEY = "vendor_id" -ACCOUNT_JOIN_KEY = "account_code" - -#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two -#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. -_AR_OPEN = "payment_state IN ('not_paid','partial')" -_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" -_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" -_CONFIRMED = "state IN ('sale','done')" - -#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. -#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that -#: left the population, which is right — a reversed invoice must not keep inflating a total. But -#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded -#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible -#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. -MAX_SHRINK = 0.5 - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _registry(): - """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the - route layer before `platform/` is necessarily on the path.""" - import core.registry as registry - return registry - - -def _iso_today(): - return _dt.date.today().strftime("%Y-%m-%d") - - -def _preset(field, flow="odoo_relational"): - """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the - grid renders it grey and the preset walls refuse a rename or a delete.""" - field = dict(field) - field["automation"] = {"flowId": flow, "preset": True} - return field - - -#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. -#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing -#: one here would be a second condition vocabulary beside `_clean_rollup`'s. -_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, - {"field": "payment_state", "op": "eq", "value": "partial"}], - "conditionConj": "or"} - - -# --------------------------------------------------------------------------------------------- -# FIELD CONTRACTS -# --------------------------------------------------------------------------------------------- -# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None -# for an unknown one — which DELETES the column silently on the next read rather than erroring. -def _scope_field(): - return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", - "source": "overlay", "default": False, - "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " - "metric in this product excludes. The row is kept so no Odoo id is " - "missing; filter on this column to reconcile against the AR page."} - - -def _refreshed_field(): - return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", - "default": False, "description": "When this row was last reconciled against Odoo."} - - -def agent_fields(): - """One row per SALES AGENT, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it - is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 - carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without - the flag and 3 are flagged with no commission yet**. Either source alone silently drops real - agents. Same shape as `read_customers`' two document universes, for the same reason. - """ - return [_preset(f) for f in ( - {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " - "their commission lines instead - both are real, which is why this " - "table is the union of the two."}, - {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", - "source": "overlay", "default": True}, - # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 - # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). - # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` - # for the ruling. ⚠ This one costs more than the other two and the difference is worth - # recording: those were a click-through to a record whose id stays on the row, while this - # was an agent's BOOK — the list of customers assigned to them. The id side survives - # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in - # the data and only the rendered list is gone; the same question is answerable on the - # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on - # `sales_lines` / `sales_orders`. - _refreshed_field(), - )] - - -def account_fields(): - """One row per `account.account` — the GL chart, the owner's "GL codes". - - ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest - of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on - expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest - binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a - link into a table that does not exist. Declaring one here would render a permanently blank - column, which is the exact trap D-87 warns about from the value site. - """ - return [_preset(f) for f in ( - {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.account` id. Also this row's id."}, - # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its - # options omit is wave-26 item 24: the filter panel answers with a list that cannot match - # what is stored. - {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", - "default": True, - "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", - "income_other", "asset_cash", "asset_current", "asset_receivable", - "asset_fixed", "asset_non_current", "asset_prepayments", - "liability_current", "liability_payable", "liability_credit_card", - "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, - {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked for the expense family - the same predicate the semantic layer's " - "gl_lines topic uses, so this column and that topic cannot disagree."}, - _refreshed_field(), - )] - - -def vendor_fields(): - """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. - - ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 - vendors, of which only 9 also appear in the customer population. Pointing bills at - `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly - empty column, not an error. - """ - return [_preset(f) for f in ( - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. - # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) - # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by - # default would push the useful ones off the first screen, and DESIGN.md's "never - # over-explain" applies to columns as much as to prose. Every one is one click away in the - # field picker, and a locked database still allows fields (item 4's vocabulary). - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": True}, - {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, - {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, - {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", - "default": False}, - {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, - "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, - # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already - # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a - # shared spelling across two linked grids is how a rollup ends up summing the wrong column. - {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, - {"key": "website", "label": "Website", "type": "url", "source": "overlay", - "default": False}, - {"key": "street", "label": "Street", "type": "text", "source": "overlay", - "default": False}, - {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", - "default": False}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, - {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, - {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, - "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def bill_fields(): - """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. - - ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 - expense GL lines. What a person calls "expenses" is both, and they are different tables - the - bill is what you pay, the line is what it was coded to. This is the payable; the line ledger - is the read-through mirror grid (R2). - """ - return [_preset(f) for f in ( - {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's - # direction, so a refund reduces a total without anybody re-deriving a sign here. - {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["in_invoice", "in_refund"]}, - {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def invoice_fields(): - """One row per POSTED customer invoice or refund — the full history, not just what is open.""" - return [_preset(f) for f in ( - {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " - "why AR rollups need no filter."}, - {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the - # table held open AR only; the full posted history also carries paid / in_payment / - # reversed. A select holding a value its options do not declare is the wave-26 item-24 - # defect — the filter panel answers with a list that cannot match what is stored. - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["out_invoice", "out_refund"]}, - _scope_field(), - # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like - # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. - # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather - # than left to be inferred from a green gate. - # - # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). - # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data - # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: - # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, - # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. - # - # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: - # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` - # prefix test relaxed — which makes EVERY GATE GREEN while - # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` - # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids - # touching that line this wave for exactly that reason. - # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was - # raised from, and the mirror did not sync it until this wave — so order->invoice was a - # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. - # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold - # anything; a merged one can hold several origins space-separated). The link resolves - # against `order_no` and finds nothing when the text is not an order name — the honest - # outcome, and the reason this is a join HINT rather than a foreign key. - {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, - {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, - _refreshed_field(), - )] - - -def order_fields(): - """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", - "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", - "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, - _scope_field(), - # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same - # column in `invoice_fields` above for the ruling and the refused alternative. This row - # keeps `customer` and `partner_id`, so only the click-through is gone. - # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS - # order, matched on the order's own name. - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, - _refreshed_field(), - )] - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE -# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two -# lists must name the SAME columns, and both failure directions are silent: -# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); -# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. -# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just -# add a column". -# -# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings -# rather than omissions: -# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw -# document, not the mirror), so a link at a read-through grain resolves against nothing and -# renders a permanently blank column — the trap `account_fields` names from the other end. -# The join ids ride as ordinary filterable columns instead, so the relationships are all -# still reachable by a person and by SQL. -# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by -# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column -# would be blank for every row forever. `section_line_grids` tolerates the key; the honest -# thing is not to declare it. - - -def order_line_fields(): - """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` - scope every wholesale metric in this product uses. - - MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded - sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — - but `done` stays in the option list because it is in the SCOPE, and a filter offering only - what happens to be stored today goes stale the first time an order is marked done. - - ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` - carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — - which is also what makes `order_no` a readable primary cell instead of a line id. - - ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a - FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the - type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both - render through the client's `numberText`, which rounds nothing without a `format.decimals` - bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so - it takes the type that does not paint one. - """ - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The sale order this line belongs to. Zero orders have a blank name, " - "which is why it is the primary cell rather than the line id."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order.line` id. Also this row's id."}, - {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", - "default": False, - "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 101 section and note lines, which carry no product."}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", - "default": False}, - {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, - "agg": "sum", - "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " - "(returns), so nothing here is rounded."}, - {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", - "default": False, "agg": "sum", - "description": "Odoo's own line margin. Populated on every line."}, - {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", - "default": False, - "description": "The cost Odoo priced this line's margin against, per unit."}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - _scope_field(), - )] - - -def gl_line_fields(): - """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the - grain a person can actually browse. - - MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 - draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are - invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the - reader chooses. That is the same decision the binding states from the SQL side. - - ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: - **61,911 lines carry no partner** (journal entries that are not about a customer), and - **1,727 carry no account** at all, which is also why `account_code` — the key - `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies - it is a LEFT one. - - ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: - five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · - `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. - A select whose options go stale answers a filter with a list that cannot match a stored value - (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were - measured COMPLETE against the whole table. - """ - return [_preset(f) for f in ( - {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The journal entry this line belongs to. Never blank."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move.line` id. Also this row's id."}, - {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", - "default": False}, - {"key": "account", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", - "default": True, - "description": "The GL code, from the joined chart of accounts — the key " - "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " - "account."}, - {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 61,911 lines that are not about a partner."}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, - {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, - {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", - "default": False, - "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, - {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", - "default": False}, - {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", - "default": True, "options": ["draft", "posted", "cancel"]}, - _scope_field(), - )] - - -def product_fields(): - """One row per `product.product`, keyed on its id — EVERY product, archived ones included. - - ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes - map to more than one product id (re-SKU / merge history). The code is what a human reads and - the id is what `sales_lines.product` groups by, so both are columns and only the id is the - identity. - - ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products - carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a - blank name — measured. Pinning `code` would give those rows a blank primary cell, which is - exactly D-80: a first column nothing populates quietly becoming the row's identity - ([[fallback-that-became-the-rule]]). - """ - return [_preset(f) for f in ( - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `product.product` id. Also this row's id, and what " - "every product rollup groups by."}, - {"key": "code", "label": "SKU", "type": "text", "source": "overlay", - "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " - "products that are not stocked SKUs."}, - {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Unticked = archived. Archived products are kept because they still " - "carry sales history — two of them sold this year."}, - # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a - # select whose options go stale answers a filter with a list that cannot match a stored - # value (wave-26 item 24). Text filters honestly and never goes out of date. - {"key": "category", "label": "Category", "type": "text", "source": "overlay", - "default": True}, - {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", - "default": False, "options": ["product", "consu", "service"]}, - {"key": "standard_price", "label": "Standard cost", "type": "currency", - "source": "overlay", "default": True}, - # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in - # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; - # `sales_lines` holds 256,810 rows that are never copied into this table. - {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "units", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -def customer_fields(): - """One row per partner Odoo has transacted with, keyed on the `res.partner` id. - - ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a - relation Odoo already decided. The rollups come in two kinds on purpose: - * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer - anything about a document the table holds, including a date rank; - * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query - over the whole mirror — they can answer a DATE-WINDOWED money question, which a link - rollup cannot, because a condition can only compare against a literal and a literal year - start is right until 1 January. - """ - return [_preset(f) for f in ( - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, - {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": False}, - {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", - "default": True, - "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " - "Customers-module convention)."}, - # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this - # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are - # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to - # itself and look plausible doing it. - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - _scope_field(), - - # --- the relations ------------------------------------------------------------------- - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", - "default": True, - "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the - # agents table — zero dangling, which is why this ships as a link rather than a lookup. - {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, - - # --- link rollups over the invoice history -------------------------------------------- - # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual - # is exactly 0, so summing the full history gives the open balance to the cent. - # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. - # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while - # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / - # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two - # numbers with one name, 4x apart, in one product is how a correct figure gets reported - # as a bug; the scope belongs in the label, not only in a column somebody has to filter. - {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "description": "Open balance across every posted document, INCLUDING the Amazon " - "channel. Filter `In wholesale scope` to reconcile with the AR page.", - "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, - {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", - "source": "overlay", "default": False, "agg": "sum", - "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, - # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every - # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right - # this module refuses everywhere else. - {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", - "source": "overlay", "default": True, - "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, - # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a - # date column finds no numbers and returns BLANK — a column that renders empty forever - # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. - {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", - "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, - - # --- link rollups over the order history ---------------------------------------------- - {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "fn": "countall"}}, - {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "field": "order_date", "fn": "latest", - "sortBy": "order_date", "sortDir": "desc"}}, - - # --- source-backed (read-through) rollups --------------------------------------------- - # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted - # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced - # flag. Different grain, different question — the metric KEY carries the distinction, - # which is the whole reason a rollup may not carry SQL of its own. - {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": False, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", - "default": False, - "rollup": {"source": {"topic": "sales_orders", "measure": "orders", - "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -# --------------------------------------------------------------------------------------------- -# READING THE MIRROR -# --------------------------------------------------------------------------------------------- -def excluded_names(): - """The partner names out of wholesale scope, from the ONE place that defines them. - - Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and - the day somebody adds a channel this module would keep answering the old question. - """ - try: - import core.odoo as odoo - names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() - return {str(n).strip().lower() for n in names if str(n).strip()} - except Exception: # noqa: BLE001 - return set() - - -def excluded_ids(cur, names=None): - """The out-of-scope partner IDS, resolved against the MIRROR. - - ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. - `account_move.partner_name` is a copy taken when the document was written, and this module's - own `customers_from` says so out loud — *"a partner's name can differ across documents - (renames land on new invoices only)"*. So a rename would put some of one partner's documents - in scope and the rest out, silently, and the totals would stop reconciling with nothing to - point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. - - ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a - LIVE `search_read`, so importing it here would make spawning four locked databases fail - whenever Odoo is unreachable — including on this developer machine, where the handshake dies - on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. - - ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which - is the correct direction. 44 transacting partners carry no name at all; treating an - unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. - """ - names = names if names is not None else excluded_names() - if not names: - return set() - rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() - return {int(pid) for pid, name in rows if str(name).strip().lower() in names} - - -def columns(cur, table): - """The column names a mirror table actually has, lowercased. `set()` if the table is absent. - - ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY - phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns - added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, - `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` - keys, which `ready()` does not read. So there is a real window, right after a boot, where the - store reports READY and a column this module names does not exist yet — and DuckDB answers a - missing identifier with a Binder error, which reached the operator as a bare `500`. - ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the - whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers - degrade the COLUMN to blank and still write every id. - """ - try: - rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") - return {str(d[0]).lower() for d in rows.description} - except Exception: # noqa: BLE001 - return set() - - -def _col(have, name, default="NULL"): - """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" - return name if str(name).split(".")[-1].lower() in have else default - - -def _as_date(value): - """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the - STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" - if not value: - return "" - return str(value)[:10] - - -def _in_scope(pid, excluded): - """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" - return "" if int(pid) in excluded else "1" - - -def read_invoices(cur, excluded=None, open_only=False): - """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. - - ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the - out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and - TAGS the channel instead. Two queries would be two populations, and they drift the moment - either is edited. - - Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - where = _AR_WHERE if open_only else _POSTED_DOCS - # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in - # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does - # not have the column yet — and DuckDB answers a missing identifier with a Binder error that - # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the - # link degrades to blank for one sync cycle instead of refusing the whole spawn. - have = columns(cur, "account_move") - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " - f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " - f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r - scope = _in_scope(pid, excluded) - if open_only and not scope: - continue - out.append({ - "_id": str(mid), - "invoice_no": str(name or ""), - "odoo_id": int(mid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "invoice_date": _as_date(inv_date), - "due_date": _as_date(due), - "residual": float(residual or 0.0), - "amount_untaxed": float(untaxed or 0.0), - "payment_state": str(pay_state or ""), - "move_type": str(mtype or ""), - "origin_order": str(origin or "").strip(), - "wholesale_scope": scope, - }) - return out - - -def read_open_ar(cur, excluded=None): - """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. - - Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an - oracle answers exactly one question. It is a projection of `read_invoices`, never a second - query. - """ - return read_invoices(cur, excluded=excluded, open_only=True) - - -def read_orders(cur, excluded=None): - """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "sale_order") - sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " - f" state, amount_untaxed, {_col(have, 'invoice_status')} " - f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r - out.append({ - "_id": str(oid), - "order_no": str(name or ""), - "odoo_id": int(oid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "order_date": _as_date(when), - "amount_untaxed": float(untaxed or 0.0), - "team": str(team or ""), - "state": str(state or ""), - "invoice_status": str(inv_status or ""), - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_products(cur): - """[(row dict)] — EVERY `product.product`, keyed on its id. - - ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they - were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products - that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three - uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product - grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to - land — silently. 119 extra rows is the whole cost of the claim being literally true. - """ - have = columns(cur, "product_product") - sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " - f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " - "FROM product_product") - out = [] - for r in cur.execute(sql).fetchall(): - (prid, code, name, categ, ptype, cost, active) = r - out.append({ - "_id": str(prid), - "product": str(name or ""), - PRODUCT_JOIN_KEY: int(prid), - "code": str(code or ""), - "active": "1" if active else "", - "category": str(categ or ""), - "product_type": str(ptype or ""), - "standard_price": float(cost or 0.0), - }) - return out - - -def read_customers(cur, excluded=None): - """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. - - The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale - order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. - - ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — - the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old - docstring said partners with no document are *"left out on purpose — a row that can never - appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same - join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** - (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer - a salesperson has not sold to yet is exactly the row a prospecting view needs. - - ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the - document legs for the rank leg would drop **16 partners that hold posted documents** (7 - archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry - `partner_id` LINKS straight back here — so those links would dangle with nothing reporting - it. When other tables point AT a population, a widening must be a SUPERSET. - - ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same - `columns()`/`_col` discipline every other optional column here uses — but note the difference - honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the - POPULATION back to the document union. It degrades to today's behaviour rather than to an - empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the - column is missing so the narrowing can never pass for done. - - ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above - states the rule for this module and it applies with more force to a POPULATION than to a name - list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its - mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this - time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the - weather. One source, always present at spawn time: the mirror. - - ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR - partners; sourcing a customer registry from its own receivables is what kept most Odoo ids - out of the store in the first place. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "res_partner") - # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join - # itself names the column, so `_col` on the SELECT list alone would still fail to bind. - agent = ("ag.name" if "agent_id" in have else "NULL") - agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") - join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") - # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an - # archived prospect out, and a rank test without it would re-admit the 47 archived partners - # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. - rank_leg = (" OR (p.customer_rank > 0 AND p.active) " - if {"customer_rank", "active"} <= have else "") - sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " - f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " - "FROM res_partner p " - f"{join}" - "WHERE p.id IN (" - f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " - " UNION " - " SELECT partner_id FROM account_move " - f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" - f"{rank_leg}") - out = [] - for r in cur.execute(sql).fetchall(): - (pid, name, city, state, country, agent, agent_id) = r - out.append({ - "_id": str(pid), - "customer": str(name or ""), - JOIN_KEY: int(pid), - "city": str(city or ""), - "state": str(state or ""), - "country": str(country or ""), - "agent": str(agent or ""), - AGENT_JOIN_KEY: int(agent_id) if agent_id else "", - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_agents(cur): - """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. - - ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: - Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list - silently becomes NULL and every row would read "not an agent" indistinguishably from - "unknown". Read it as a truth value, never as a presence test. - """ - have = columns(cur, "res_partner") - if "id" not in have: - return [] - flagged = "p.agent" if "agent" in have else "FALSE" - # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that - # distinction is the whole point of this block. `columns()` was written for a missing COLUMN - # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that - # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent - # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table - # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is - # precisely D-107's shape, and it would have arrived on the first deploy of this feature. - # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the - # commission table the population falls back to the FLAGGED partners alone and `commissioned` - # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. - has_comm = bool(columns(cur, "account_invoice_line_agent")) - commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " - " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") - union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " - " UNION " if has_comm else "") - sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " - "FROM res_partner p WHERE p.id IN (" - f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") - out = [] - for (aid, name, flag, comm) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(aid), - "agent": str(name or ""), - "odoo_id": int(aid), - AGENT_JOIN_KEY: int(aid), - "flagged": "1" if flag else "", - "commissioned": "1" if comm else "", - }) - return out - - -def read_accounts(cur): - """[(row dict)] — the whole GL chart, keyed on the `account.account` id. - - ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: - `harness/semantic.py`'s `gl_lines` topic scopes expenses as - `account_type in ('expense','expense_depreciation')`. A second definition here is how a - column and a topic start disagreeing about the same word. - ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so - there is nothing to filter and every account is a row. - """ - have = columns(cur, "account_account") - if not have: - return [] - sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " - f" {_col(have, 'name', chr(39) + chr(39))}, " - f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") - out = [] - for (aid, code, name, atype) in cur.execute(sql).fetchall(): - t = str(atype or "") - out.append({ - "_id": str(aid), - ACCOUNT_JOIN_KEY: str(code or ""), - "account_name": str(name or ""), - "odoo_id": int(aid), - "account_type": t, - "is_expense": "1" if t in ("expense", "expense_depreciation") else "", - }) - return out - - -_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" - - -def read_bills(cur): - """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" - have = columns(cur, "account_move") - if not have: - return [] - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - f" {_col(have, 'amount_untaxed_signed', '0')}, " - f" {_col(have, 'amount_residual_signed', '0')}, " - f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " - f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r - out.append({ - "_id": str(mid), - "bill_no": str(name or ""), - "odoo_id": int(mid), - "vendor": str(pname or ""), - VENDOR_JOIN_KEY: int(pid), - "invoice_date": _as_date(when), - "due_date": _as_date(due), - "amount_untaxed": float(untaxed or 0.0), - "residual": float(residual or 0.0), - "payment_state": str(pay or ""), - "move_type": str(mtype or ""), - }) - return out - - -def read_vendors(cur): - """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. - - ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its - invoices. The asymmetry is intentional and the reason is what that function's own comment - says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. - There is no second document universe for vendors — a partner with no bill has no payable - history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills - carry a null partner; all 393 vendors resolve in `res_partner`). - """ - have = columns(cur, "res_partner") - if not have or not columns(cur, "account_move"): - return [] - # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured - # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a - # vendor record for — how to reach them, who they are for tax, and where they are. - # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the - # column. That is not defensive habit: this projection has to keep working against a mirror - # that has not been re-synced since the widening, and the alternative is a reader that raises - # on the exact box the widening was meant to help. The columns arrive when the backfill runs; - # until then these read blank rather than failing. - blank = chr(39) + chr(39) - sql = ("SELECT p.id, p.name, " - f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " - f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " - f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " - f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " - f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " - f"{_col(have, 'p.zip', blank)} " - "FROM res_partner p WHERE p.id IN " - f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " - " AND partner_id IS NOT NULL)") - out = [] - for (pid, name, country, email, phone, mobile, website, vat, ref, - street, street2, city, zipc) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(pid), - "vendor": str(name or ""), - "odoo_id": int(pid), - VENDOR_JOIN_KEY: int(pid), - "country": str(country or ""), - "email": str(email or ""), - "phone": str(phone or ""), - "mobile": str(mobile or ""), - "website": str(website or ""), - "vat": str(vat or ""), - "vendor_ref": str(ref or ""), - "street": str(street or ""), - "street2": str(street2 or ""), - "city": str(city or ""), - "zip": str(zipc or ""), - }) - return out - - -def customers_from(invoice_rows): - """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. - - ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a - pure function over rows and the gate uses it to prove the FOLD against a fixture without a - mirror; deleting it would cost a test its independence from the SQL. - """ - out = {} - for row in invoice_rows: - pid = row[JOIN_KEY] - entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], - JOIN_KEY: pid}) - # A partner's name can differ across documents (renames land on new invoices only); - # the newest non-empty one wins so the locked table shows what Odoo shows today. - if row["customer"]: - entry["customer"] = row["customer"] - return list(out.values()) - - -# --------------------------------------------------------------------------------------------- -# THE SPAWN -# --------------------------------------------------------------------------------------------- -class Refused(Exception): - """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have - been written wrong.""" - - -#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole -#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. -#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. -TABLES = ( - # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / - # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules - # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current - # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it - # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries - # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). - # Dropping the row here is what stops them being planned, built or re-created; - # `RETIRED_KEYS` below is what removes the rows a tenant already has. - ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), - ("orders", ORDERS_KEY, "Odoo orders", order_fields), - # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders - # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" - # is four more spec rows and four readers rather than a new substrate. - ("agents", AGENTS_KEY, "Odoo agents", agent_fields), - ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), - ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), - ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), - # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that - # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) - # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of - # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` - # does not name, so the grids would be bound to the mirror and unreachable, which is this - # wave's own [[reachable-is-not-the-same-as-built]] shape. - ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), - ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), -) - -#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` -#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. -#: -#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: -#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so -#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that -#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the -#: key, and this is the shape that says so. -#: -#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all -#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the -#: database, never over which Odoo model was queried. Same for `account_move`, which is the -#: customer-invoice book under one WHERE and the vendor-bill book under another. -TABLE_SUBJECTS = { - # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects - # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` - # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one - # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future - # spec row tried to bring it back. - INVOICES_KEY: "odoo:account.move.customer", - ORDERS_KEY: "odoo:sale.order", - AGENTS_KEY: "odoo:res.partner.agent", - ACCOUNTS_KEY: "odoo:account.account", - VENDORS_KEY: "odoo:res.partner.vendor", - BILLS_KEY: "odoo:account.move.vendor", - ORDER_LINES_KEY: "odoo:sale.order.line", - GL_LINES_KEY: "odoo:account.move.line", -} - -#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. -#: -#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the -#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the -#: owner actually sees in the nav. This set is what removes them, and it is applied inside -#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every -#: resync. -#: -#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a -#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms -#: it — and the running Space reverts it within a minute, because the container holds the document -#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a -#: script is a dry run that reports success. -#: -#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job -#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This -#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, -#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes -#: and re-adds the table every 30 minutes. -RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) - -#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. -#: -#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness -#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was -#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote -#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new -#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned -#: the gate RED until this constant went with them. It did, and that is the ratchet working. -#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. - -#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no -#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a -#: policy `plan()` could forget. The two line grains are absent for that reason and no other. -_READERS = { - "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), - "products": lambda cur, excluded: read_products(cur), - "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), - "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), - "agents": lambda cur, excluded: read_agents(cur), - "accounts": lambda cur, excluded: read_accounts(cur), - "vendors": lambda cur, excluded: read_vendors(cur), - "bills": lambda cur, excluded: read_bills(cur), -} - -#: The table keys this module can never materialise — DERIVED from the absence of a reader, never -#: typed out, so it cannot drift from the fact it describes. -#: -#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE -#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a -#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a -#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps -#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early -#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is -#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the -#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. -#: The conversion writes the stamp; a table that needs no conversion still needs the statement. -READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) - - -class _LentDoc: - """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. - - ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` - resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE - 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per - table per loop, so passing the live handle would have added ~16 full document copies to a - function that already reads it exactly once — and with `st=None` (the gate's fixture posture, - and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset - fetch per table, on a path that has no business touching the network at all. - `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` - takes `st` rather than `defn`, so the lending happens one level up, here. - - ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim - that quietly proxied other keys would be a second store with a partial view, which is worse - than one that says what it knows. - """ - - def __init__(self, doc, key): - self._doc, self._key = doc if isinstance(doc, dict) else {}, key - - def get(self, name): - return self._doc if name == self._key else None - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner opened the Odoo connector and found nothing to configure: no key, no server database, -# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all -# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey -# it — a config the route knows and the sync path does not is a switch that flips nothing. -# -#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` -CONFIG_KEY = "odoo_connector_config" - -#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The -#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a -#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. -#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None -#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather -#: than a silent once-a-century schedule. -SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} -SYNC_FLOOR_SECONDS = 1800 -DEFAULT_SYNC = "30m" - - -def read_config(rt): - """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not - make a connector look disconnected.""" - try: - cur = rt.get(CONFIG_KEY) if rt is not None else None - except Exception: # noqa: BLE001 - cur = None - cur = cur if isinstance(cur, dict) else {} - grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} - every = cur.get("syncEvery") - return {"grids": {str(k): bool(v) for k, v in grids.items()}, - "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, - "frozen": bool(cur.get("frozen")), - "frozenAt": str(cur.get("frozenAt") or "")} - - -def grid_choices(rt): - """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from - `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). - - ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is - what they have today; only an explicit untick turns one off. The alternative — an empty - config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. - """ - chosen = read_config(rt)["grids"] - return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} - for _bucket, key, label, _fields in TABLES] - - -def enabled_buckets(rt): - """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" - chosen = read_config(rt)["grids"] - return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} - - -def sync_seconds(rt): - """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). - - ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that - predates the preset list, or one written by any path that is not the route, must still not be - able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that - holds until somebody finds the second way in [[limit-with-no-enforcer]]. - """ - secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) - if secs is None: - return None - return max(int(secs), SYNC_FLOOR_SECONDS) - - -def frozen(rt): - """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and - stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" - return bool(read_config(rt)["frozen"]) - - -def plan(cur, rt=None): - """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. - - Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and - so **every cap is checked before anything is committed**. - - Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty - list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — - `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's - second sentence carried as data rather than left for a reader to infer from an empty list. - - ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn - bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would - create some and refuse the rest — leaving a locked invoices database with no rollup host, - while the route answered as though nothing had happened. A partial spawn is worse than a - refused one, so the count is checked against the tables that ALREADY exist, before the first - write. `rt` also carries the stored row counts the shrink guard compares against. - """ - ut = _ut() - # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from - # three places in falling authority: the registry, a stored `connected: True`, then the - # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a - # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one - # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently - # not apply on exactly the run that creates the databases. This module DECLARES these keys, so - # it is the honest place to say what they are. Idempotent (a set add), and it fills the - # evaluator's input rather than becoming a second evaluator. - ut.register_connected(*[key for _b, key, _l, _f in TABLES]) - excluded = excluded_ids(cur) - # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is - # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = - # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this - # function as the caller that reads it, which is the seam working as designed — one evaluator, - # so the spawn, the write doors and the wire cannot disagree about whether a table is capped - # ([[one-evaluator-per-question]]). - # - # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: - # * no reader at all — structural, permanent, and the case that must not depend on a store - # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); - # * a reader exists but the table has already been converted to read-through — the - # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them - # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` - # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table - # D-87's conversion had already emptied. - # - # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used - # to be read after the build loop; it moved up because `row_limit` needs it and reading it per - # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face - # fetch per table on a path that must never touch the network). - existing = {} - if rt is not None: - try: - existing = dict(rt.get(ut.STORE_KEY) or {}) - except Exception: # noqa: BLE001 - existing = {} - # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and - # without it R6's report is silently absent on the run that matters most — the FIRST spawn. - # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so - # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and - # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids - # were skipped with no stated reason. The structural `reader is None` guard still did its job; - # what went missing was the half of R6 that has to SAY WHY. - # - # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` - # would work in one line, and `core.user_tables` explicitly reserves that registrar — - # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for - # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold - # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could - # be eligible FOR), but the law is written without an exception, so this lends the evaluator - # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so - # what is lent is the document as it stands the moment this plan applies. - lent = _LentDoc({**existing, - **{k: {**(existing.get(k) or {}), "readThrough": True} - for k in READ_THROUGH_KEYS}}, - ut.STORE_KEY) - - built, limits = {}, {} - caps = {} - for bucket, key, _label, _fields in TABLES: - reader = _READERS.get(bucket) - caps[key] = cap = ut.row_limit(key, st=lent) - if reader is None or cap == 0: - built[bucket] = [] - report = ut.limit_report(key, st=lent) - if report: - limits[key] = report - continue - built[bucket] = reader(cur, excluded) - problems = [] - - if rt is not None: - needed = [k for _b, k, _l, _f in TABLES if k not in existing] - if needed and len(existing) + len(needed) > ut.MAX_TABLES: - problems.append( - f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " - f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " - f"part of a linked set") - - for bucket, key, _label, _fields in TABLES: - rows = built[bucket] - cap = caps[key] # asked ONCE per table, above — never re-read per loop - # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED - # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in - # how many data from the API source (as long as its from a connected source like Odoo) that - # can be pulled into the app… Now if there is lag or it can't be done, you need to - # explicitly tell me why and recommend a fix."* Both halves are here: a connected table - # answers `None` and is never refused for its size, and anything that IS still bounded is - # reported with its cause and its recommendation instead of a hand-typed line. - # - # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: - # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and - # the loop above already handed it an empty list. Neither may reach the refusal. The - # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped - # table understates every total it feeds while looking exactly like a complete one. - if cap and len(rows) > cap: - report = ut.limit_report(key, st=lent) or {} - limits[key] = report - problems.append( - f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " - f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " - f"{report.get('recommendation', '')}".strip()) - # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every - # spawn after the first conversion: zero rows against a stored population is the INTENDED - # end state there, not the partial mirror read this guard exists to catch. - if cap == 0: - continue - stored = len(((existing.get(key) or {}).get("rows")) or {}) - if stored and len(rows) < stored * MAX_SHRINK: - problems.append( - f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " - f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " - f"shrinking; refusing rather than deleting rows that still exist") - built["problems"] = problems - # R6's second sentence as DATA rather than prose: every table whose rows this plan did not - # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. - # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that - # is not a bucket name is inert there, exactly as `problems` has always been. - built["limits"] = limits - return built - - -def apply_plan(rt, built, username="automation", today=None, report=None): - """Create-or-merge every table in `built` and its rows. Idempotent by construction. - - Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the - same record — which is also what makes "every Odoo unique id is in the database" a checkable - statement rather than a hopeful one. - - ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. - """ - if built.get("problems"): - raise Refused("; ".join(built["problems"])) - stamp = today or _iso_today() - written = {} - # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and - # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. - retired = [] - plans = [(key, label, fields(), built[bucket]) - for bucket, key, label, fields in TABLES if bucket in built] - - # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. - # - # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full - # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB - # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits - # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). - # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. - # - # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at - # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had - # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to - # create. Inside one updater, a raise aborts before anything is persisted. - def _apply_all(cur): - cur = cur if isinstance(cur, dict) else {} - # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE - # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every - # resync), which is the only place a change to the tenant document survives (D-195: the - # same edit from a CLI reports success and is reverted within a minute). - # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot - # contain them, and there is no door here that could re-add one. `removed` is reported so - # a caller can SAY what happened rather than infer it from a table going missing. - # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them - # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its - # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die - # with `TypeError: list indices must be integers`. One dict, two value shapes, is a - # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through - # its OWN channel and the return type stays exactly what it was. - for key in RETIRED_KEYS: - if cur.pop(key, None) is not None: - retired.append(key) - for key, label, fields, rows in plans: - written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _apply_all, flush="sync") - if report is not None and retired: - report["retired"] = sorted(retired) - return written - - -def _ensure_table(rt, key, label, fields, rows, username, stamp): - """One table, written on its own. Kept because the gate drives a single table directly, and - because a caller with one table to reconcile should not have to compose an updater.""" - written = {} - - def _one(cur): - cur = cur if isinstance(cur, dict) else {} - written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _one, flush="sync") - return written["counts"] - - -def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): - """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. - - ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — - but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` - was built FROM open invoices, removal meant a customer who paid their bill vanished from the - registry; now a partner leaves only when their last document does. The shrink guard in - `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. - """ - ut = _ut() - wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} - for row in wanted.values(): - row["refreshed"] = stamp - counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} - subject = TABLE_SUBJECTS.get(key) - table = cur.get(key) - if table is None: - if len(cur) >= ut.MAX_TABLES: - # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a - # successful refresh over a table that does not exist. - raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") - # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. - # - # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This - # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key - # on every pass. A claim test outside this `if` refuses the table it created last boot, - # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would - # take the product down to prevent a duplicate that already exists. Refusing the SECOND - # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to - # remove, and it is removed by dropping its `TABLES` row, not by a guard here. - # - # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. - # Never a module-level cache: one Space process serves every tenant, and two tenants both - # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated - # here). A stored definition with no `subject` claims nothing. - held = {t["subject"]: k for k, t in cur.items() - if isinstance(t, dict) and t.get("subject")} - # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered - # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. - other = _registry().subject_conflict(subject, key, claimed=held) - if other: - raise Refused( - f"{key}: refusing to create a second database for {subject!r} — {other!r} " - f"already presents it. One subject, one database (item 12a); if this table is " - f"meant to replace {other!r}, retire {other!r} first rather than shipping both") - table = cur[key] = { - "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, - "createdBy": username, "created": stamp, "fields": [], "rows": {}, - # recordMode = a LOCKED database (item-3 nomenclature): no human may add or - # delete records, while fields stay addable. Odoo owns this population. - "recordMode": ut.AUTOMATION_RECORD_MODE, - } - table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) - # W30-T35 — the durable "my rows are not in this document" statement, on the tables no - # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not - # `setdefault`: it is derived from the code's own structure, so the code is what it must agree - # with, and a definition that somehow lost the flag should regain it rather than keep serving - # an empty grid. - if key in READ_THROUGH_KEYS: - table["readThrough"] = True - # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same - # reason `readThrough` is: it is derived from this module's own structure, so the code is what - # it has to agree with, and a definition that lost the stamp should regain it rather than go - # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees - # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold - # process reading a fresh document has nothing else to read the claim off. - if subject: - table["subject"] = subject - have = {str(f.get("key")): f for f in (table.get("fields") or [])} - for field in fields: - # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. - # - # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND - # patch, so every field that goes through the normal door has one. This function does NOT - # go through that door — it writes definitions straight into the document — and - # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that - # ever omitted it would 500 the entire rows route rather than degrade one column. Measured - # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is - # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that - # prompted it came from a hand-written test fixture, not from production). - # - # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a - # missing `source` means the definition does not say which stratum owns the column, and - # rendering it as blank would bury that. The default matches what the validator would - # have stamped, so the bypass stops being a hole without inventing a second rule. - if isinstance(field, dict) and not field.get("source"): - field = {**field, "source": "overlay"} - fkey = str(field.get("key")) - if fkey not in have: - table.setdefault("fields", []).append(dict(field)) - continue - # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The - # widening moved `payment_state`'s option list and every rollup's conditions; a - # create-only merge would have left the LIVE table declaring the old contract - # forever, so the column would render but its filter could not match what is stored. - # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a - # column a user added to a locked database is never rewritten. - stored = have[fkey] - # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, - # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the - # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on - # an Odoo database would silently revert at the next boot rebuild. - if _ut().user_edited(stored): - continue - if (stored.get("automation") or {}).get("preset"): - for prop in ("label", "type", "options", "link", "rollup", "description", - "agg", "pinned", "default"): - if prop in field: - stored[prop] = field[prop] - else: - stored.pop(prop, None) - stored_rows = table.setdefault("rows", {}) - for rid, values in wanted.items(): - current = stored_rows.get(rid) - if current is None: - stored_rows[rid] = dict(values) - counts["added"] += 1 - elif any(str(current.get(k, "")) != str(v) for k, v in values.items() - if k != "refreshed"): - current.update(values) - counts["updated"] += 1 - else: - current["refreshed"] = values["refreshed"] - for rid in [r for r in stored_rows if r not in wanted]: - stored_rows.pop(rid, None) - counts["removed"] += 1 - return counts - - -def is_royal(tenant): - return str(tenant or "").strip().lower() in RI_SLUGS - - -def refresh(rt, tenant, username="automation", cur=None, today=None): - """THE entry point — the store-resync path and the route both call this. - - ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row - still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than - silently authoritative. - """ - if not is_royal(tenant): - raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " - f"Imports only); refusing to spawn empty locked databases") - # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, - # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is - # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process - # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a - # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked - # databases from another customer's rows — a spawn, not a read, so the wrong numbers become - # durable. Entitlement is not residency. - # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot - # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. - if rt is not None: - rt.assert_datastore_matches() - # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. - # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing - # the credential alone is not a freeze — this function is also reached by the boot rebuild and - # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a - # disconnected workspace would silently re-materialise from `.env` on the next tick and the - # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: - # the resync loop calling this every cycle must not be handed an exception as a status. - if rt is not None and frozen(rt): - return {"tables": {}, "frozen": True, - "note": "this workspace has disconnected Odoo; its databases are frozen as " - "static data and are not being refreshed"} - if cur is None: - from harness import datastore - cur = datastore.ro_con() - built = plan(cur, rt=rt) - # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the - # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a - # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so - # every cap, refusal and limit report is still computed over the full set — a config must not - # be able to hide a problem by hiding the table that has it. - # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from - # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not - # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. - if rt is not None: - keep = enabled_buckets(rt) - skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) - for bucket, _key, _l, _f in TABLES: - if bucket not in keep: - built.pop(bucket, None) - else: - skipped = [] - # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. - # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no - # longer built) SAYS SO, with the keys. - plan_report = {} - written = apply_plan(rt, built, username=username, today=today, report=plan_report) - return {"tables": written, - **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), - # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. - **({"skipped": skipped} if skipped else {}), - **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} +"""odoo_relational.py — Odoo entities as LOCKED relational databases. + +Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset +Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually +replace. + +⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the +owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports +tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the +partners who owed money — so most Odoo ids were simply absent, and the missing rows were the +reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: + + ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a + posted customer document + ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code + ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund + ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order + +⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's +own `plan()` refused above it — but the cap was never a property of the store (see the measured +banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). +The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until +wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory +budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). +The four tables together are +17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split +is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); +they are answered by the read-through rollup, which never copies a row. + +⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts +GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables +dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and +carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner +holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. +Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you +filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps +excluding, because `modules/ar` is its oracle and an oracle answers ONE question. + +⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over +ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate +**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside +`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the +`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the +wider link would count 31,418 documents and call them open invoices, so those two rollups carry +the oracle's predicate as an explicit `payment_state` condition pair. +""" +import datetime as _dt + +#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: +#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. +RI_SLUGS = ("", "royal-imports") + +INVOICES_KEY = "ut_odoo_invoices" +CUSTOMERS_KEY = "ut_odoo_customers" +ORDERS_KEY = "ut_odoo_orders" +PRODUCTS_KEY = "ut_odoo_products" +#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, +#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to +#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). +AGENTS_KEY = "ut_odoo_agents" +ACCOUNTS_KEY = "ut_odoo_accounts" +BILLS_KEY = "ut_odoo_bills" +VENDORS_KEY = "ut_odoo_vendors" + +#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. +#: +#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it +#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced +#: rather than left standing beside its own contradiction. The obstacle was never the number of +#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED +#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) +#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 +#: settles what that means: *"there is no cap in how many data from the API source … can be pulled +#: into the app"*, so the answer is a different residency, never a bigger ceiling. +#: +#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the +#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this +#: database stores no rows HERE", which is a different statement from `None` ("connected and +#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator +#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work +#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with +#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working +#: grid; that is the whole shape of the conversion. +ORDER_LINES_KEY = "ut_odoo_order_lines" +GL_LINES_KEY = "ut_odoo_gl_lines" + +#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). +JOIN_KEY = "partner_id" +#: The product-grain equivalent. +PRODUCT_JOIN_KEY = "product_id" +#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but +#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the +#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. +AGENT_JOIN_KEY = "agent_id" +#: A vendor is also a `res.partner`; same reasoning, its own column. +VENDOR_JOIN_KEY = "vendor_id" +ACCOUNT_JOIN_KEY = "account_code" + +#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two +#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. +_AR_OPEN = "payment_state IN ('not_paid','partial')" +_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" +_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" +_CONFIRMED = "state IN ('sale','done')" + +#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. +#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that +#: left the population, which is right — a reversed invoice must not keep inflating a total. But +#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded +#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible +#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. +MAX_SHRINK = 0.5 + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _registry(): + """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the + route layer before `platform/` is necessarily on the path.""" + import core.registry as registry + return registry + + +def _iso_today(): + return _dt.date.today().strftime("%Y-%m-%d") + + +def _preset(field, flow="odoo_relational"): + """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the + grid renders it grey and the preset walls refuse a rename or a delete.""" + field = dict(field) + field["automation"] = {"flowId": flow, "preset": True} + return field + + +#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. +#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing +#: one here would be a second condition vocabulary beside `_clean_rollup`'s. +_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, + {"field": "payment_state", "op": "eq", "value": "partial"}], + "conditionConj": "or"} + + +# --------------------------------------------------------------------------------------------- +# FIELD CONTRACTS +# --------------------------------------------------------------------------------------------- +# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None +# for an unknown one — which DELETES the column silently on the next read rather than erroring. +def _scope_field(): + return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", + "source": "overlay", "default": False, + "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " + "metric in this product excludes. The row is kept so no Odoo id is " + "missing; filter on this column to reconcile against the AR page."} + + +def _refreshed_field(): + return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", + "default": False, "description": "When this row was last reconciled against Odoo."} + + +def agent_fields(): + """One row per SALES AGENT, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it + is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 + carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without + the flag and 3 are flagged with no commission yet**. Either source alone silently drops real + agents. Same shape as `read_customers`' two document universes, for the same reason. + """ + return [_preset(f) for f in ( + {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " + "their commission lines instead - both are real, which is why this " + "table is the union of the two."}, + {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", + "source": "overlay", "default": True}, + # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 + # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). + # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` + # for the ruling. ⚠ This one costs more than the other two and the difference is worth + # recording: those were a click-through to a record whose id stays on the row, while this + # was an agent's BOOK — the list of customers assigned to them. The id side survives + # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in + # the data and only the rendered list is gone; the same question is answerable on the + # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on + # `sales_lines` / `sales_orders`. + _refreshed_field(), + )] + + +def account_fields(): + """One row per `account.account` — the GL chart, the owner's "GL codes". + + ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest + of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on + expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest + binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a + link into a table that does not exist. Declaring one here would render a permanently blank + column, which is the exact trap D-87 warns about from the value site. + """ + return [_preset(f) for f in ( + {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.account` id. Also this row's id."}, + # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its + # options omit is wave-26 item 24: the filter panel answers with a list that cannot match + # what is stored. + {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", + "default": True, + "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", + "income_other", "asset_cash", "asset_current", "asset_receivable", + "asset_fixed", "asset_non_current", "asset_prepayments", + "liability_current", "liability_payable", "liability_credit_card", + "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, + {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked for the expense family - the same predicate the semantic layer's " + "gl_lines topic uses, so this column and that topic cannot disagree."}, + _refreshed_field(), + )] + + +def vendor_fields(): + """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. + + ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 + vendors, of which only 9 also appear in the customer population. Pointing bills at + `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly + empty column, not an error. + """ + return [_preset(f) for f in ( + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. + # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) + # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by + # default would push the useful ones off the first screen, and DESIGN.md's "never + # over-explain" applies to columns as much as to prose. Every one is one click away in the + # field picker, and a locked database still allows fields (item 4's vocabulary). + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": True}, + {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, + {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, + {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", + "default": False}, + {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, + "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, + # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already + # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a + # shared spelling across two linked grids is how a rollup ends up summing the wrong column. + {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, + {"key": "website", "label": "Website", "type": "url", "source": "overlay", + "default": False}, + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, + {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, + "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def bill_fields(): + """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. + + ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 + expense GL lines. What a person calls "expenses" is both, and they are different tables - the + bill is what you pay, the line is what it was coded to. This is the payable; the line ledger + is the read-through mirror grid (R2). + """ + return [_preset(f) for f in ( + {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's + # direction, so a refund reduces a total without anybody re-deriving a sign here. + {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["in_invoice", "in_refund"]}, + {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def invoice_fields(): + """One row per POSTED customer invoice or refund — the full history, not just what is open.""" + return [_preset(f) for f in ( + {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " + "why AR rollups need no filter."}, + {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the + # table held open AR only; the full posted history also carries paid / in_payment / + # reversed. A select holding a value its options do not declare is the wave-26 item-24 + # defect — the filter panel answers with a list that cannot match what is stored. + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["out_invoice", "out_refund"]}, + _scope_field(), + # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like + # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. + # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather + # than left to be inferred from a green gate. + # + # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). + # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data + # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: + # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, + # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. + # + # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: + # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` + # prefix test relaxed — which makes EVERY GATE GREEN while + # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` + # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids + # touching that line this wave for exactly that reason. + # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was + # raised from, and the mirror did not sync it until this wave — so order->invoice was a + # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. + # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold + # anything; a merged one can hold several origins space-separated). The link resolves + # against `order_no` and finds nothing when the text is not an order name — the honest + # outcome, and the reason this is a join HINT rather than a foreign key. + {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, + {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, + _refreshed_field(), + )] + + +def order_fields(): + """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", + "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", + "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, + _scope_field(), + # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same + # column in `invoice_fields` above for the ruling and the refused alternative. This row + # keeps `customer` and `partner_id`, so only the click-through is gone. + # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS + # order, matched on the order's own name. + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, + _refreshed_field(), + )] + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE +# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two +# lists must name the SAME columns, and both failure directions are silent: +# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); +# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. +# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just +# add a column". +# +# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings +# rather than omissions: +# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw +# document, not the mirror), so a link at a read-through grain resolves against nothing and +# renders a permanently blank column — the trap `account_fields` names from the other end. +# The join ids ride as ordinary filterable columns instead, so the relationships are all +# still reachable by a person and by SQL. +# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by +# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column +# would be blank for every row forever. `section_line_grids` tolerates the key; the honest +# thing is not to declare it. + + +def order_line_fields(): + """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` + scope every wholesale metric in this product uses. + + MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded + sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — + but `done` stays in the option list because it is in the SCOPE, and a filter offering only + what happens to be stored today goes stale the first time an order is marked done. + + ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` + carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — + which is also what makes `order_no` a readable primary cell instead of a line id. + + ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a + FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the + type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both + render through the client's `numberText`, which rounds nothing without a `format.decimals` + bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so + it takes the type that does not paint one. + """ + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The sale order this line belongs to. Zero orders have a blank name, " + "which is why it is the primary cell rather than the line id."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order.line` id. Also this row's id."}, + {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", + "default": False, + "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 101 section and note lines, which carry no product."}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", + "default": False}, + {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, + "agg": "sum", + "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " + "(returns), so nothing here is rounded."}, + {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", + "default": False, "agg": "sum", + "description": "Odoo's own line margin. Populated on every line."}, + {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", + "default": False, + "description": "The cost Odoo priced this line's margin against, per unit."}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + _scope_field(), + )] + + +def gl_line_fields(): + """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the + grain a person can actually browse. + + MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 + draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are + invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the + reader chooses. That is the same decision the binding states from the SQL side. + + ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: + **61,911 lines carry no partner** (journal entries that are not about a customer), and + **1,727 carry no account** at all, which is also why `account_code` — the key + `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies + it is a LEFT one. + + ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: + five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · + `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. + A select whose options go stale answers a filter with a list that cannot match a stored value + (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were + measured COMPLETE against the whole table. + """ + return [_preset(f) for f in ( + {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The journal entry this line belongs to. Never blank."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move.line` id. Also this row's id."}, + {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", + "default": False}, + {"key": "account", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", + "default": True, + "description": "The GL code, from the joined chart of accounts — the key " + "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " + "account."}, + {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 61,911 lines that are not about a partner."}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, + {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, + {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", + "default": False, + "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, + {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", + "default": False}, + {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", + "default": True, "options": ["draft", "posted", "cancel"]}, + _scope_field(), + )] + + +def product_fields(): + """One row per `product.product`, keyed on its id — EVERY product, archived ones included. + + ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes + map to more than one product id (re-SKU / merge history). The code is what a human reads and + the id is what `sales_lines.product` groups by, so both are columns and only the id is the + identity. + + ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products + carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a + blank name — measured. Pinning `code` would give those rows a blank primary cell, which is + exactly D-80: a first column nothing populates quietly becoming the row's identity + ([[fallback-that-became-the-rule]]). + """ + return [_preset(f) for f in ( + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `product.product` id. Also this row's id, and what " + "every product rollup groups by."}, + {"key": "code", "label": "SKU", "type": "text", "source": "overlay", + "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " + "products that are not stocked SKUs."}, + {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Unticked = archived. Archived products are kept because they still " + "carry sales history — two of them sold this year."}, + # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a + # select whose options go stale answers a filter with a list that cannot match a stored + # value (wave-26 item 24). Text filters honestly and never goes out of date. + {"key": "category", "label": "Category", "type": "text", "source": "overlay", + "default": True}, + {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", + "default": False, "options": ["product", "consu", "service"]}, + {"key": "standard_price", "label": "Standard cost", "type": "currency", + "source": "overlay", "default": True}, + # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in + # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; + # `sales_lines` holds 256,810 rows that are never copied into this table. + {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "units", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +def customer_fields(): + """One row per partner Odoo has transacted with, keyed on the `res.partner` id. + + ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a + relation Odoo already decided. The rollups come in two kinds on purpose: + * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer + anything about a document the table holds, including a date rank; + * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query + over the whole mirror — they can answer a DATE-WINDOWED money question, which a link + rollup cannot, because a condition can only compare against a literal and a literal year + start is right until 1 January. + """ + return [_preset(f) for f in ( + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, + {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": False}, + {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", + "default": True, + "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " + "Customers-module convention)."}, + # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this + # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are + # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to + # itself and look plausible doing it. + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + _scope_field(), + + # --- the relations ------------------------------------------------------------------- + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", + "default": True, + "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the + # agents table — zero dangling, which is why this ships as a link rather than a lookup. + {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, + + # --- link rollups over the invoice history -------------------------------------------- + # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual + # is exactly 0, so summing the full history gives the open balance to the cent. + # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. + # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while + # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / + # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two + # numbers with one name, 4x apart, in one product is how a correct figure gets reported + # as a bug; the scope belongs in the label, not only in a column somebody has to filter. + {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "description": "Open balance across every posted document, INCLUDING the Amazon " + "channel. Filter `In wholesale scope` to reconcile with the AR page.", + "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, + {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", + "source": "overlay", "default": False, "agg": "sum", + "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, + # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every + # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right + # this module refuses everywhere else. + {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", + "source": "overlay", "default": True, + "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, + # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a + # date column finds no numbers and returns BLANK — a column that renders empty forever + # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. + {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", + "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, + + # --- link rollups over the order history ---------------------------------------------- + {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "fn": "countall"}}, + {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "field": "order_date", "fn": "latest", + "sortBy": "order_date", "sortDir": "desc"}}, + + # --- source-backed (read-through) rollups --------------------------------------------- + # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted + # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced + # flag. Different grain, different question — the metric KEY carries the distinction, + # which is the whole reason a rollup may not carry SQL of its own. + {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": False, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", + "default": False, + "rollup": {"source": {"topic": "sales_orders", "measure": "orders", + "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +# --------------------------------------------------------------------------------------------- +# READING THE MIRROR +# --------------------------------------------------------------------------------------------- +def excluded_names(): + """The partner names out of wholesale scope, from the ONE place that defines them. + + Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and + the day somebody adds a channel this module would keep answering the old question. + """ + try: + import core.odoo as odoo + names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() + return {str(n).strip().lower() for n in names if str(n).strip()} + except Exception: # noqa: BLE001 + return set() + + +def excluded_ids(cur, names=None): + """The out-of-scope partner IDS, resolved against the MIRROR. + + ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. + `account_move.partner_name` is a copy taken when the document was written, and this module's + own `customers_from` says so out loud — *"a partner's name can differ across documents + (renames land on new invoices only)"*. So a rename would put some of one partner's documents + in scope and the rest out, silently, and the totals would stop reconciling with nothing to + point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. + + ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a + LIVE `search_read`, so importing it here would make spawning four locked databases fail + whenever Odoo is unreachable — including on this developer machine, where the handshake dies + on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. + + ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which + is the correct direction. 44 transacting partners carry no name at all; treating an + unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. + """ + names = names if names is not None else excluded_names() + if not names: + return set() + rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() + return {int(pid) for pid, name in rows if str(name).strip().lower() in names} + + +def columns(cur, table): + """The column names a mirror table actually has, lowercased. `set()` if the table is absent. + + ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY + phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns + added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, + `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` + keys, which `ready()` does not read. So there is a real window, right after a boot, where the + store reports READY and a column this module names does not exist yet — and DuckDB answers a + missing identifier with a Binder error, which reached the operator as a bare `500`. + ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the + whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers + degrade the COLUMN to blank and still write every id. + """ + try: + rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") + return {str(d[0]).lower() for d in rows.description} + except Exception: # noqa: BLE001 + return set() + + +def _col(have, name, default="NULL"): + """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" + return name if str(name).split(".")[-1].lower() in have else default + + +def _as_date(value): + """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the + STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" + if not value: + return "" + return str(value)[:10] + + +def _in_scope(pid, excluded): + """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" + return "" if int(pid) in excluded else "1" + + +def read_invoices(cur, excluded=None, open_only=False): + """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. + + ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the + out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and + TAGS the channel instead. Two queries would be two populations, and they drift the moment + either is edited. + + Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + where = _AR_WHERE if open_only else _POSTED_DOCS + # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in + # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does + # not have the column yet — and DuckDB answers a missing identifier with a Binder error that + # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the + # link degrades to blank for one sync cycle instead of refusing the whole spawn. + have = columns(cur, "account_move") + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " + f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " + f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r + scope = _in_scope(pid, excluded) + if open_only and not scope: + continue + out.append({ + "_id": str(mid), + "invoice_no": str(name or ""), + "odoo_id": int(mid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "invoice_date": _as_date(inv_date), + "due_date": _as_date(due), + "residual": float(residual or 0.0), + "amount_untaxed": float(untaxed or 0.0), + "payment_state": str(pay_state or ""), + "move_type": str(mtype or ""), + "origin_order": str(origin or "").strip(), + "wholesale_scope": scope, + }) + return out + + +def read_open_ar(cur, excluded=None): + """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. + + Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an + oracle answers exactly one question. It is a projection of `read_invoices`, never a second + query. + """ + return read_invoices(cur, excluded=excluded, open_only=True) + + +def read_orders(cur, excluded=None): + """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "sale_order") + sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " + f" state, amount_untaxed, {_col(have, 'invoice_status')} " + f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r + out.append({ + "_id": str(oid), + "order_no": str(name or ""), + "odoo_id": int(oid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "order_date": _as_date(when), + "amount_untaxed": float(untaxed or 0.0), + "team": str(team or ""), + "state": str(state or ""), + "invoice_status": str(inv_status or ""), + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_products(cur): + """[(row dict)] — EVERY `product.product`, keyed on its id. + + ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they + were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products + that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three + uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product + grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to + land — silently. 119 extra rows is the whole cost of the claim being literally true. + """ + have = columns(cur, "product_product") + sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " + f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " + "FROM product_product") + out = [] + for r in cur.execute(sql).fetchall(): + (prid, code, name, categ, ptype, cost, active) = r + out.append({ + "_id": str(prid), + "product": str(name or ""), + PRODUCT_JOIN_KEY: int(prid), + "code": str(code or ""), + "active": "1" if active else "", + "category": str(categ or ""), + "product_type": str(ptype or ""), + "standard_price": float(cost or 0.0), + }) + return out + + +def read_customers(cur, excluded=None): + """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. + + The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale + order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. + + ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — + the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old + docstring said partners with no document are *"left out on purpose — a row that can never + appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same + join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** + (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer + a salesperson has not sold to yet is exactly the row a prospecting view needs. + + ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the + document legs for the rank leg would drop **16 partners that hold posted documents** (7 + archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry + `partner_id` LINKS straight back here — so those links would dangle with nothing reporting + it. When other tables point AT a population, a widening must be a SUPERSET. + + ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same + `columns()`/`_col` discipline every other optional column here uses — but note the difference + honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the + POPULATION back to the document union. It degrades to today's behaviour rather than to an + empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the + column is missing so the narrowing can never pass for done. + + ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above + states the rule for this module and it applies with more force to a POPULATION than to a name + list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its + mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this + time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the + weather. One source, always present at spawn time: the mirror. + + ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR + partners; sourcing a customer registry from its own receivables is what kept most Odoo ids + out of the store in the first place. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "res_partner") + # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join + # itself names the column, so `_col` on the SELECT list alone would still fail to bind. + agent = ("ag.name" if "agent_id" in have else "NULL") + agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") + join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") + # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an + # archived prospect out, and a rank test without it would re-admit the 47 archived partners + # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. + rank_leg = (" OR (p.customer_rank > 0 AND p.active) " + if {"customer_rank", "active"} <= have else "") + sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " + f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " + "FROM res_partner p " + f"{join}" + "WHERE p.id IN (" + f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " + " UNION " + " SELECT partner_id FROM account_move " + f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" + f"{rank_leg}") + out = [] + for r in cur.execute(sql).fetchall(): + (pid, name, city, state, country, agent, agent_id) = r + out.append({ + "_id": str(pid), + "customer": str(name or ""), + JOIN_KEY: int(pid), + "city": str(city or ""), + "state": str(state or ""), + "country": str(country or ""), + "agent": str(agent or ""), + AGENT_JOIN_KEY: int(agent_id) if agent_id else "", + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_agents(cur): + """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. + + ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: + Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list + silently becomes NULL and every row would read "not an agent" indistinguishably from + "unknown". Read it as a truth value, never as a presence test. + """ + have = columns(cur, "res_partner") + if "id" not in have: + return [] + flagged = "p.agent" if "agent" in have else "FALSE" + # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that + # distinction is the whole point of this block. `columns()` was written for a missing COLUMN + # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that + # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent + # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table + # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is + # precisely D-107's shape, and it would have arrived on the first deploy of this feature. + # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the + # commission table the population falls back to the FLAGGED partners alone and `commissioned` + # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. + has_comm = bool(columns(cur, "account_invoice_line_agent")) + commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " + " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") + union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " + " UNION " if has_comm else "") + sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " + "FROM res_partner p WHERE p.id IN (" + f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") + out = [] + for (aid, name, flag, comm) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(aid), + "agent": str(name or ""), + "odoo_id": int(aid), + AGENT_JOIN_KEY: int(aid), + "flagged": "1" if flag else "", + "commissioned": "1" if comm else "", + }) + return out + + +def read_accounts(cur): + """[(row dict)] — the whole GL chart, keyed on the `account.account` id. + + ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: + `harness/semantic.py`'s `gl_lines` topic scopes expenses as + `account_type in ('expense','expense_depreciation')`. A second definition here is how a + column and a topic start disagreeing about the same word. + ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so + there is nothing to filter and every account is a row. + """ + have = columns(cur, "account_account") + if not have: + return [] + sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " + f" {_col(have, 'name', chr(39) + chr(39))}, " + f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") + out = [] + for (aid, code, name, atype) in cur.execute(sql).fetchall(): + t = str(atype or "") + out.append({ + "_id": str(aid), + ACCOUNT_JOIN_KEY: str(code or ""), + "account_name": str(name or ""), + "odoo_id": int(aid), + "account_type": t, + "is_expense": "1" if t in ("expense", "expense_depreciation") else "", + }) + return out + + +_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" + + +def read_bills(cur): + """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" + have = columns(cur, "account_move") + if not have: + return [] + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + f" {_col(have, 'amount_untaxed_signed', '0')}, " + f" {_col(have, 'amount_residual_signed', '0')}, " + f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " + f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r + out.append({ + "_id": str(mid), + "bill_no": str(name or ""), + "odoo_id": int(mid), + "vendor": str(pname or ""), + VENDOR_JOIN_KEY: int(pid), + "invoice_date": _as_date(when), + "due_date": _as_date(due), + "amount_untaxed": float(untaxed or 0.0), + "residual": float(residual or 0.0), + "payment_state": str(pay or ""), + "move_type": str(mtype or ""), + }) + return out + + +def read_vendors(cur): + """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. + + ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its + invoices. The asymmetry is intentional and the reason is what that function's own comment + says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. + There is no second document universe for vendors — a partner with no bill has no payable + history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills + carry a null partner; all 393 vendors resolve in `res_partner`). + """ + have = columns(cur, "res_partner") + if not have or not columns(cur, "account_move"): + return [] + # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured + # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a + # vendor record for — how to reach them, who they are for tax, and where they are. + # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the + # column. That is not defensive habit: this projection has to keep working against a mirror + # that has not been re-synced since the widening, and the alternative is a reader that raises + # on the exact box the widening was meant to help. The columns arrive when the backfill runs; + # until then these read blank rather than failing. + blank = chr(39) + chr(39) + sql = ("SELECT p.id, p.name, " + f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " + f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " + f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " + f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " + f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " + f"{_col(have, 'p.zip', blank)} " + "FROM res_partner p WHERE p.id IN " + f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " + " AND partner_id IS NOT NULL)") + out = [] + for (pid, name, country, email, phone, mobile, website, vat, ref, + street, street2, city, zipc) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(pid), + "vendor": str(name or ""), + "odoo_id": int(pid), + VENDOR_JOIN_KEY: int(pid), + "country": str(country or ""), + "email": str(email or ""), + "phone": str(phone or ""), + "mobile": str(mobile or ""), + "website": str(website or ""), + "vat": str(vat or ""), + "vendor_ref": str(ref or ""), + "street": str(street or ""), + "street2": str(street2 or ""), + "city": str(city or ""), + "zip": str(zipc or ""), + }) + return out + + +def customers_from(invoice_rows): + """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. + + ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a + pure function over rows and the gate uses it to prove the FOLD against a fixture without a + mirror; deleting it would cost a test its independence from the SQL. + """ + out = {} + for row in invoice_rows: + pid = row[JOIN_KEY] + entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], + JOIN_KEY: pid}) + # A partner's name can differ across documents (renames land on new invoices only); + # the newest non-empty one wins so the locked table shows what Odoo shows today. + if row["customer"]: + entry["customer"] = row["customer"] + return list(out.values()) + + +# --------------------------------------------------------------------------------------------- +# THE SPAWN +# --------------------------------------------------------------------------------------------- +class Refused(Exception): + """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have + been written wrong.""" + + +#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole +#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. +#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. +TABLES = ( + # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / + # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules + # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current + # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it + # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries + # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). + # Dropping the row here is what stops them being planned, built or re-created; + # `RETIRED_KEYS` below is what removes the rows a tenant already has. + ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), + ("orders", ORDERS_KEY, "Odoo orders", order_fields), + # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders + # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" + # is four more spec rows and four readers rather than a new substrate. + ("agents", AGENTS_KEY, "Odoo agents", agent_fields), + ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), + ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), + ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), + # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that + # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) + # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of + # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` + # does not name, so the grids would be bound to the mirror and unreachable, which is this + # wave's own [[reachable-is-not-the-same-as-built]] shape. + ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), + ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), +) + +#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` +#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. +#: +#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: +#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so +#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that +#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the +#: key, and this is the shape that says so. +#: +#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all +#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the +#: database, never over which Odoo model was queried. Same for `account_move`, which is the +#: customer-invoice book under one WHERE and the vendor-bill book under another. +TABLE_SUBJECTS = { + # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects + # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` + # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one + # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future + # spec row tried to bring it back. + INVOICES_KEY: "odoo:account.move.customer", + ORDERS_KEY: "odoo:sale.order", + AGENTS_KEY: "odoo:res.partner.agent", + ACCOUNTS_KEY: "odoo:account.account", + VENDORS_KEY: "odoo:res.partner.vendor", + BILLS_KEY: "odoo:account.move.vendor", + ORDER_LINES_KEY: "odoo:sale.order.line", + GL_LINES_KEY: "odoo:account.move.line", +} + +#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. +#: +#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the +#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the +#: owner actually sees in the nav. This set is what removes them, and it is applied inside +#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every +#: resync. +#: +#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a +#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms +#: it — and the running Space reverts it within a minute, because the container holds the document +#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a +#: script is a dry run that reports success. +#: +#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job +#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This +#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, +#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes +#: and re-adds the table every 30 minutes. +RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) + +#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. +#: +#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness +#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was +#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote +#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new +#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned +#: the gate RED until this constant went with them. It did, and that is the ratchet working. +#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. + +#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no +#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a +#: policy `plan()` could forget. The two line grains are absent for that reason and no other. +_READERS = { + "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), + "products": lambda cur, excluded: read_products(cur), + "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), + "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), + "agents": lambda cur, excluded: read_agents(cur), + "accounts": lambda cur, excluded: read_accounts(cur), + "vendors": lambda cur, excluded: read_vendors(cur), + "bills": lambda cur, excluded: read_bills(cur), +} + +#: The table keys this module can never materialise — DERIVED from the absence of a reader, never +#: typed out, so it cannot drift from the fact it describes. +#: +#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE +#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a +#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a +#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps +#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early +#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is +#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the +#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. +#: The conversion writes the stamp; a table that needs no conversion still needs the statement. +READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) + + +class _LentDoc: + """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. + + ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` + resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE + 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per + table per loop, so passing the live handle would have added ~16 full document copies to a + function that already reads it exactly once — and with `st=None` (the gate's fixture posture, + and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset + fetch per table, on a path that has no business touching the network at all. + `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` + takes `st` rather than `defn`, so the lending happens one level up, here. + + ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim + that quietly proxied other keys would be a second store with a partial view, which is worse + than one that says what it knows. + """ + + def __init__(self, doc, key): + self._doc, self._key = doc if isinstance(doc, dict) else {}, key + + def get(self, name): + return self._doc if name == self._key else None + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner opened the Odoo connector and found nothing to configure: no key, no server database, +# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all +# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey +# it — a config the route knows and the sync path does not is a switch that flips nothing. +# +#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` +CONFIG_KEY = "odoo_connector_config" + +#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The +#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a +#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. +#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None +#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather +#: than a silent once-a-century schedule. +SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} +SYNC_FLOOR_SECONDS = 1800 +DEFAULT_SYNC = "30m" + + +def read_config(rt): + """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not + make a connector look disconnected.""" + try: + cur = rt.get(CONFIG_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + cur = None + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} + every = cur.get("syncEvery") + return {"grids": {str(k): bool(v) for k, v in grids.items()}, + "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, + "frozen": bool(cur.get("frozen")), + "frozenAt": str(cur.get("frozenAt") or "")} + + +def grid_choices(rt): + """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from + `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). + + ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is + what they have today; only an explicit untick turns one off. The alternative — an empty + config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. + """ + chosen = read_config(rt)["grids"] + return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} + for _bucket, key, label, _fields in TABLES] + + +def enabled_buckets(rt): + """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" + chosen = read_config(rt)["grids"] + return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} + + +def sync_seconds(rt): + """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). + + ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that + predates the preset list, or one written by any path that is not the route, must still not be + able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that + holds until somebody finds the second way in [[limit-with-no-enforcer]]. + """ + secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) + if secs is None: + return None + return max(int(secs), SYNC_FLOOR_SECONDS) + + +def frozen(rt): + """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and + stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" + return bool(read_config(rt)["frozen"]) + + +def plan(cur, rt=None): + """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. + + Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and + so **every cap is checked before anything is committed**. + + Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty + list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — + `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's + second sentence carried as data rather than left for a reader to infer from an empty list. + + ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn + bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would + create some and refuse the rest — leaving a locked invoices database with no rollup host, + while the route answered as though nothing had happened. A partial spawn is worse than a + refused one, so the count is checked against the tables that ALREADY exist, before the first + write. `rt` also carries the stored row counts the shrink guard compares against. + """ + ut = _ut() + # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from + # three places in falling authority: the registry, a stored `connected: True`, then the + # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a + # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one + # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently + # not apply on exactly the run that creates the databases. This module DECLARES these keys, so + # it is the honest place to say what they are. Idempotent (a set add), and it fills the + # evaluator's input rather than becoming a second evaluator. + ut.register_connected(*[key for _b, key, _l, _f in TABLES]) + excluded = excluded_ids(cur) + # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is + # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = + # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this + # function as the caller that reads it, which is the seam working as designed — one evaluator, + # so the spawn, the write doors and the wire cannot disagree about whether a table is capped + # ([[one-evaluator-per-question]]). + # + # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: + # * no reader at all — structural, permanent, and the case that must not depend on a store + # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); + # * a reader exists but the table has already been converted to read-through — the + # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them + # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` + # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table + # D-87's conversion had already emptied. + # + # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used + # to be read after the build loop; it moved up because `row_limit` needs it and reading it per + # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face + # fetch per table on a path that must never touch the network). + existing = {} + if rt is not None: + try: + existing = dict(rt.get(ut.STORE_KEY) or {}) + except Exception: # noqa: BLE001 + existing = {} + # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and + # without it R6's report is silently absent on the run that matters most — the FIRST spawn. + # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so + # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and + # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids + # were skipped with no stated reason. The structural `reader is None` guard still did its job; + # what went missing was the half of R6 that has to SAY WHY. + # + # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` + # would work in one line, and `core.user_tables` explicitly reserves that registrar — + # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for + # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold + # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could + # be eligible FOR), but the law is written without an exception, so this lends the evaluator + # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so + # what is lent is the document as it stands the moment this plan applies. + lent = _LentDoc({**existing, + **{k: {**(existing.get(k) or {}), "readThrough": True} + for k in READ_THROUGH_KEYS}}, + ut.STORE_KEY) + + built, limits = {}, {} + caps = {} + for bucket, key, _label, _fields in TABLES: + reader = _READERS.get(bucket) + caps[key] = cap = ut.row_limit(key, st=lent) + if reader is None or cap == 0: + built[bucket] = [] + report = ut.limit_report(key, st=lent) + if report: + limits[key] = report + continue + built[bucket] = reader(cur, excluded) + problems = [] + + if rt is not None: + needed = [k for _b, k, _l, _f in TABLES if k not in existing] + if needed and len(existing) + len(needed) > ut.MAX_TABLES: + problems.append( + f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " + f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " + f"part of a linked set") + + for bucket, key, _label, _fields in TABLES: + rows = built[bucket] + cap = caps[key] # asked ONCE per table, above — never re-read per loop + # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED + # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in + # how many data from the API source (as long as its from a connected source like Odoo) that + # can be pulled into the app… Now if there is lag or it can't be done, you need to + # explicitly tell me why and recommend a fix."* Both halves are here: a connected table + # answers `None` and is never refused for its size, and anything that IS still bounded is + # reported with its cause and its recommendation instead of a hand-typed line. + # + # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: + # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and + # the loop above already handed it an empty list. Neither may reach the refusal. The + # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped + # table understates every total it feeds while looking exactly like a complete one. + if cap and len(rows) > cap: + report = ut.limit_report(key, st=lent) or {} + limits[key] = report + problems.append( + f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " + f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " + f"{report.get('recommendation', '')}".strip()) + # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every + # spawn after the first conversion: zero rows against a stored population is the INTENDED + # end state there, not the partial mirror read this guard exists to catch. + if cap == 0: + continue + stored = len(((existing.get(key) or {}).get("rows")) or {}) + if stored and len(rows) < stored * MAX_SHRINK: + problems.append( + f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " + f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " + f"shrinking; refusing rather than deleting rows that still exist") + built["problems"] = problems + # R6's second sentence as DATA rather than prose: every table whose rows this plan did not + # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. + # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that + # is not a bucket name is inert there, exactly as `problems` has always been. + built["limits"] = limits + return built + + +def apply_plan(rt, built, username="automation", today=None, report=None): + """Create-or-merge every table in `built` and its rows. Idempotent by construction. + + Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the + same record — which is also what makes "every Odoo unique id is in the database" a checkable + statement rather than a hopeful one. + + ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. + """ + if built.get("problems"): + raise Refused("; ".join(built["problems"])) + stamp = today or _iso_today() + written = {} + # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and + # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. + retired = [] + plans = [(key, label, fields(), built[bucket]) + for bucket, key, label, fields in TABLES if bucket in built] + + # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. + # + # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full + # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB + # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits + # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). + # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. + # + # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at + # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had + # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to + # create. Inside one updater, a raise aborts before anything is persisted. + def _apply_all(cur): + cur = cur if isinstance(cur, dict) else {} + # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE + # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every + # resync), which is the only place a change to the tenant document survives (D-195: the + # same edit from a CLI reports success and is reverted within a minute). + # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot + # contain them, and there is no door here that could re-add one. `removed` is reported so + # a caller can SAY what happened rather than infer it from a table going missing. + # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them + # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its + # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die + # with `TypeError: list indices must be integers`. One dict, two value shapes, is a + # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through + # its OWN channel and the return type stays exactly what it was. + for key in RETIRED_KEYS: + if cur.pop(key, None) is not None: + retired.append(key) + for key, label, fields, rows in plans: + written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _apply_all, flush="sync") + if report is not None and retired: + report["retired"] = sorted(retired) + return written + + +def _ensure_table(rt, key, label, fields, rows, username, stamp): + """One table, written on its own. Kept because the gate drives a single table directly, and + because a caller with one table to reconcile should not have to compose an updater.""" + written = {} + + def _one(cur): + cur = cur if isinstance(cur, dict) else {} + written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _one, flush="sync") + return written["counts"] + + +def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): + """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. + + ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — + but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` + was built FROM open invoices, removal meant a customer who paid their bill vanished from the + registry; now a partner leaves only when their last document does. The shrink guard in + `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. + """ + ut = _ut() + wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} + for row in wanted.values(): + row["refreshed"] = stamp + counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} + subject = TABLE_SUBJECTS.get(key) + table = cur.get(key) + if table is None: + if len(cur) >= ut.MAX_TABLES: + # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a + # successful refresh over a table that does not exist. + raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") + # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. + # + # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This + # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key + # on every pass. A claim test outside this `if` refuses the table it created last boot, + # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would + # take the product down to prevent a duplicate that already exists. Refusing the SECOND + # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to + # remove, and it is removed by dropping its `TABLES` row, not by a guard here. + # + # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. + # Never a module-level cache: one Space process serves every tenant, and two tenants both + # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated + # here). A stored definition with no `subject` claims nothing. + held = {t["subject"]: k for k, t in cur.items() + if isinstance(t, dict) and t.get("subject")} + # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered + # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. + other = _registry().subject_conflict(subject, key, claimed=held) + if other: + raise Refused( + f"{key}: refusing to create a second database for {subject!r} — {other!r} " + f"already presents it. One subject, one database (item 12a); if this table is " + f"meant to replace {other!r}, retire {other!r} first rather than shipping both") + table = cur[key] = { + "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, + "createdBy": username, "created": stamp, "fields": [], "rows": {}, + # recordMode = a LOCKED database (item-3 nomenclature): no human may add or + # delete records, while fields stay addable. Odoo owns this population. + "recordMode": ut.AUTOMATION_RECORD_MODE, + } + table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) + # W30-T35 — the durable "my rows are not in this document" statement, on the tables no + # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not + # `setdefault`: it is derived from the code's own structure, so the code is what it must agree + # with, and a definition that somehow lost the flag should regain it rather than keep serving + # an empty grid. + if key in READ_THROUGH_KEYS: + table["readThrough"] = True + # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same + # reason `readThrough` is: it is derived from this module's own structure, so the code is what + # it has to agree with, and a definition that lost the stamp should regain it rather than go + # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees + # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold + # process reading a fresh document has nothing else to read the claim off. + if subject: + table["subject"] = subject + have = {str(f.get("key")): f for f in (table.get("fields") or [])} + for field in fields: + # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. + # + # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND + # patch, so every field that goes through the normal door has one. This function does NOT + # go through that door — it writes definitions straight into the document — and + # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that + # ever omitted it would 500 the entire rows route rather than degrade one column. Measured + # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is + # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that + # prompted it came from a hand-written test fixture, not from production). + # + # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a + # missing `source` means the definition does not say which stratum owns the column, and + # rendering it as blank would bury that. The default matches what the validator would + # have stamped, so the bypass stops being a hole without inventing a second rule. + if isinstance(field, dict) and not field.get("source"): + field = {**field, "source": "overlay"} + fkey = str(field.get("key")) + if fkey not in have: + table.setdefault("fields", []).append(dict(field)) + continue + # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The + # widening moved `payment_state`'s option list and every rollup's conditions; a + # create-only merge would have left the LIVE table declaring the old contract + # forever, so the column would render but its filter could not match what is stored. + # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a + # column a user added to a locked database is never rewritten. + stored = have[fkey] + # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, + # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the + # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on + # an Odoo database would silently revert at the next boot rebuild. + if _ut().user_edited(stored): + continue + if (stored.get("automation") or {}).get("preset"): + for prop in ("label", "type", "options", "link", "rollup", "description", + "agg", "pinned", "default"): + if prop in field: + stored[prop] = field[prop] + else: + stored.pop(prop, None) + stored_rows = table.setdefault("rows", {}) + for rid, values in wanted.items(): + current = stored_rows.get(rid) + if current is None: + stored_rows[rid] = dict(values) + counts["added"] += 1 + elif any(str(current.get(k, "")) != str(v) for k, v in values.items() + if k != "refreshed"): + current.update(values) + counts["updated"] += 1 + else: + current["refreshed"] = values["refreshed"] + for rid in [r for r in stored_rows if r not in wanted]: + stored_rows.pop(rid, None) + counts["removed"] += 1 + return counts + + +def is_royal(tenant): + return str(tenant or "").strip().lower() in RI_SLUGS + + +def refresh(rt, tenant, username="automation", cur=None, today=None): + """THE entry point — the store-resync path and the route both call this. + + ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row + still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than + silently authoritative. + """ + if not is_royal(tenant): + raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " + f"Imports only); refusing to spawn empty locked databases") + # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, + # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is + # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process + # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a + # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked + # databases from another customer's rows — a spawn, not a read, so the wrong numbers become + # durable. Entitlement is not residency. + # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot + # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. + if rt is not None: + rt.assert_datastore_matches() + # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. + # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing + # the credential alone is not a freeze — this function is also reached by the boot rebuild and + # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a + # disconnected workspace would silently re-materialise from `.env` on the next tick and the + # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: + # the resync loop calling this every cycle must not be handed an exception as a status. + if rt is not None and frozen(rt): + return {"tables": {}, "frozen": True, + "note": "this workspace has disconnected Odoo; its databases are frozen as " + "static data and are not being refreshed"} + if cur is None: + from harness import datastore + cur = datastore.ro_con() + built = plan(cur, rt=rt) + # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the + # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a + # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so + # every cap, refusal and limit report is still computed over the full set — a config must not + # be able to hide a problem by hiding the table that has it. + # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from + # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not + # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. + if rt is not None: + keep = enabled_buckets(rt) + skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) + for bucket, _key, _l, _f in TABLES: + if bucket not in keep: + built.pop(bucket, None) + else: + skipped = [] + # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. + # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no + # longer built) SAYS SO, with the keys. + plan_report = {} + written = apply_plan(rt, built, username=username, today=today, report=plan_report) + return {"tables": written, + **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), + # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. + **({"skipped": skipped} if skipped else {}), + **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} diff --git a/api/providers.py b/api/providers.py index 46db502b900cea6d9bece95beda04696cc1beb5f..9f08e9e7365f120b1a9b1a887c1b49e71ca137b4 100644 --- a/api/providers.py +++ b/api/providers.py @@ -124,7 +124,7 @@ PROVIDERS: dict[str, Provider] = { "likes + comments + shares + saves, and play_count inline (DECLARED " "no-empties-or-zeros, UNPROVEN until one live pull)"), "tt_comments": Capability(True, _COST_BRIGHTDATA, - "separate paid dataset, opt-in — 17 fields"), + "separate paid dataset, opt-in: 17 fields"), }), "apify": Provider( key="apify", label="Apify", key_env="AIOS_APIFY_KEY", @@ -132,7 +132,7 @@ PROVIDERS: dict[str, Provider] = { # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. "ig_post_views": Capability(True, _COST_APIFY, - "videoPlayCount — matches Instagram's displayed views"), + "videoPlayCount, matches Instagram's displayed views"), "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), }), @@ -240,12 +240,12 @@ def run(capability, work, satisfied=None, log=None): secs = round(time.time() - started, 2) if note: attempts.append(Attempt(provider.key, False, note, 0, secs)) - log(f" {provider.label}: {note} — falling through") + log(f" {provider.label}: {note}, falling through") continue if not ok(result): attempts.append(Attempt(provider.key, False, "answered without the field asked for", 0, secs)) - log(f" {provider.label}: answered, but not with what was asked for — falling through") + log(f" {provider.label}: answered, but not with what was asked for, falling through") last = result if last is None else last continue attempts.append(Attempt(provider.key, True, "", _count(result), secs)) @@ -282,6 +282,323 @@ def wire(): } + + +# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ +# +# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the +# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: +# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this +# shape'."* +# +# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the +# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is +# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a +# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. +# +# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is +# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is +# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half +# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the +# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both +# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. +# +# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that +# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. +# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling +# turn at all, so the guess never has to be made. + +#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: +#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen +#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short +#: enough that a top-up is picked up without a restart. +CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) + +#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a +#: latency optimisation, not a billing record. A second container learns the same thing from its +#: own first 402, and neither one can be wrong for longer than the cooldown. +_NO_CREDIT: dict[str, float] = {} + + +@dataclass(frozen=True) +class LlmProvider: + """One chat-completions endpoint, and what it is DECLARED able to do.""" + name: str + label: str + env: str + url: str + model: str + #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, + #: which is a different body, a different auth header and a different result shape. + wire: str + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.env) or "").strip()) + + def can(self, capability: str) -> bool: + cap = self.caps.get(capability) + return bool(cap and cap.capable and self.configured()) + + +#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put +#: cerebras first *"because this path needs tool calling and cerebras carries this account's +#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always +#: works, and the others are the cheap seats it falls through to. +LLM_PROVIDERS: dict[str, LlmProvider] = { + "anthropic": LlmProvider( + name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", + url="https://api.anthropic.com/v1/messages", + # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with + # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier + # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", + wire="anthropic", + caps={ + # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a + # typed `tool_use` content block carrying parsed `input`. There is nothing to recover + # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to + # apologise for on the other wire. + "llm_tool_calling": Capability(True, 0.0, + "typed tool_use content block; no text recovery path"), + "llm_chat": Capability(True, 0.0, "Messages API"), + "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), + }), + "cerebras": LlmProvider( + name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", + url="https://api.cerebras.ai/v1/chat/completions", + model="gpt-oss-120b", wire="openai", + caps={ + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured: this account's tool-capable " + "model per the wave-32 ladder note"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "groq": LlmProvider( + name="groq", label="Groq", env="GROQ_API_KEY", + url="https://api.groq.com/openai/v1/chat/completions", + model="llama-3.3-70b-versatile", wire="openai", + caps={ + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured. ⚠ `routes_query._FAILED_GEN` " + "exists because SOME provider on this wire answers 400 " + "with the call in `failed_generation`; the repo never " + "recorded which. Flip this to False the day it is"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "openrouter": LlmProvider( + name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o-mini", wire="openai", + caps={ + "llm_tool_calling": Capability(True, 0.0, "DECLARED, not measured"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), +} + +LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") + + +def mark_no_credit(name, seconds=None): + """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). + + ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for + that to live: a stateless ladder re-tries the empty account on every single turn, pays its + round trip, and shows the reader a longer error each time. This is that memo. + """ + _NO_CREDIT[str(name)] = time.time() + float( + CREDIT_COOLDOWN_S if seconds is None else seconds) + return _NO_CREDIT[str(name)] + + +def no_credit(name): + """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" + until = _NO_CREDIT.get(str(name)) + if not until: + return False + if time.time() >= until: + _NO_CREDIT.pop(str(name), None) + return False + return True + + +def clear_credit_memo(name=None): + """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" + if name is None: + _NO_CREDIT.clear() + else: + _NO_CREDIT.pop(str(name), None) + + +def llm_chain(capability="llm_tool_calling"): + """The provider order for `capability` — declaration first, credit memo second. + + Three filters, in this order, and each removes a DIFFERENT kind of row: + 1. `can()` — declared capable AND configured. An incapable row is never offered, so a + turn cannot be spent discovering it (the `chain()` rule, one layer up). + 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over + until the memo expires. + 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment + can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` + carries for the scraping side. + """ + raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() + names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) + return [LLM_PROVIDERS[n] for n in names + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + + +#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND +#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner +#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. +_STATUS_WORDS = { + 401: "{label} would not accept our key", + 403: "{label} would not accept our key", + 402: "{label} is out of credit", + 404: "{label} does not offer the model we asked it for", + 408: "{label} took too long", + 413: "the question was too long for {label}", + 429: "{label} is rate limiting us right now", +} + +#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent +#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the +#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and +#: it only ever decides whether to SKIP a provider, never whether to trust one. +_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", + "quota exceeded", "billing", "payment required", "add credits") + + +def is_credit_failure(status, body=""): + """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" + if int(status or 0) == 402: + return True + if int(status or 0) not in (400, 403, 429): + return False + return any(word in str(body or "").lower() for word in _CREDIT_WORDS) + + +def refusal_sentence(name, status, body=""): + """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. + + ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key + prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a + credit problem", and what reaches the caller is this module's own wording. + """ + label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label + if is_credit_failure(status, body): + return f"{label} is out of credit" + code = int(status or 0) + if code in _STATUS_WORDS: + return _STATUS_WORDS[code].format(label=label) + if 500 <= code <= 599: + return f"{label} is having trouble at their end" + return f"{label} did not answer" + + +# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ +# +# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. +# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation +# drafter) both need it, and the owner quoted an error from EACH of them in one breath: +# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool +# calls are not wired for this shape"*. Two implementations of one wire is +# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the +# ladder that declares the rung. +# +# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: +# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message +# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` +# 3. `temperature` and friends are REMOVED on the current model family +# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string +# +# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on +# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits +# it, the assistant runs on the Opus tier and sends it. + +#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. +ANTHROPIC_VERSION = "2023-06-01" + + +def anthropic_request(*, model, key, system, messages, tools, max_tokens, + tool_choice="auto", effort=None): + """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. + + `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted + out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, + re-addressed rather than re-derived, so a schema change happens in one place. + """ + system_text = "\n\n".join(str(m.get("content") or "") for m in messages + if m.get("role") == "system") + if system: + system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) + turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), + "content": str(m.get("content") or "")} + for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + body = { + "model": str(model), + "max_tokens": int(max_tokens), + "system": system_text, + "messages": turns, + "tools": [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])], + "tool_choice": {"type": "any" if tool_choice in ("required", "any") else "auto"}, + } + if effort: + body["output_config"] = {"effort": str(effort)} + return {"url": LLM_PROVIDERS["anthropic"].url, + "headers": {"x-api-key": str(key), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + "json": body} + + +def anthropic_read(body): + """`(text, tool_input, refusal)` out of a Messages API answer. + + ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with + `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` + unconditionally breaks on exactly the turn a person most needs explained. + ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and + no regex recovering a call out of a 400, which is what the other wire needs. + """ + body = body if isinstance(body, dict) else {} + if str(body.get("stop_reason") or "") == "refusal": + return "", None, "the assistant declined to answer that one" + blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] + text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() + calls = [b for b in blocks if b.get("type") == "tool_use"] + got = calls[0].get("input") if calls else None + return text, (got if isinstance(got, dict) else None), None + + +def llm_status(capability="llm_tool_calling"): + """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. + + ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read + THIS, so a model offered in one place cannot be missing from the other, and neither can offer a + provider the ladder would refuse to call [[permitted-is-not-answerable]]. + """ + rows = [] + for name in LLM_DEFAULT_ORDER: + p = LLM_PROVIDERS[name] + cap = p.caps.get(capability) + rows.append({ + "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, + "configured": p.configured(), + "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), + "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), + "capable": bool(cap and cap.capable), + "outOfCredit": no_credit(name), + "note": (cap.note if cap else ""), + }) + return rows + + # ============================================================================================= # THE CANONICAL SCHEMA — owner ruling 2026-08-08: # *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set diff --git a/api/routes_admin.py b/api/routes_admin.py index ef67141b4d5798b50d4bc12dfeaee77f2c15eb0a..7b00877cbab54cc67aa66f1f09b47a9f733677f4 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -1,1051 +1,1141 @@ -"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). - -These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` -modals, over the SAME `core/users` account store — so both front-ends administer one set of -accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. - -ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit -dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record -is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. - -⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` -has no caller anywhere in the shipped app — module grants are only settable by editing the store -JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both -asserted in `verify_api.py` section A: - - * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin - clearing every checkbox to lock an account down would grant it everything. - * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to - `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose - whole point is strict isolation. - -Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in -`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the -WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both -read semantics are untouched — the seam is write-strict / read-unchanged. - -⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` -one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no -accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". -""" -import os -import re - -from fastapi import APIRouter, Body, Depends, Response - -import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag -import core.registry as registry -import core.store as store - -from deps import Session, err, perms, require_session, users -from routes_auth import _public_user - -router = APIRouter(prefix="/api/v1") - -#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under -#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no -#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. -_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") -#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog -#: enforces nothing; this does, and the two are allowed to differ because only one of them is -#: reachable from the internet. -_MIN_PW = 8 -_ROLES = ("user", "admin") - - -def admin_gate(session: Session = Depends(require_session)) -> Session: - """401 without a session, 403 without the admin role. Role, not a module grant — mirroring - `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - return session - - -def _require_store(): - """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" - try: - ok = store.available() - except Exception: - ok = False - if not ok: - raise err(503, "store_unavailable", - "the tenant store is unavailable — accounts cannot be read or changed") - - -def _registry(): - _require_store() - try: - reg = users.registry() or {} - except Exception: - raise err(503, "store_unavailable", "the tenant store is unavailable") - return reg - - -def _view(uname, rec): - """What an administrator may see about an account. NEVER `salt` or `hash` — this function is - the only projection these routes use, so there is one place that can leak and it does not.""" - return {"username": uname, - "name": rec.get("name") or uname, - "role": rec.get("role", "user"), - "bus": rec.get("bus", "all"), - "bu_labels": perms.allowed_bu_labels(rec), - "modules": rec.get("modules", "all"), - "agent": rec.get("agent") or None, - "email": rec.get("email") or None, - "tenant": _tenant_of(rec), - "active": bool(rec.get("active", True)), - # The session-revocation handle. An admin needs it: it is the only visible evidence - # that "sign them out everywhere" actually happened. - "epoch": int(rec.get("epoch") or 0), - # S3 ask 4 — one sentence for the roster's Access column, so the list does not need - # a /perms round trip per row to fill one cell. Computed from the same record the - # editor will open, so the two cannot disagree. - "accessSummary": _access_summary(rec), - "perms_v": int(rec.get("perms_v") or 0)} - - -def _access_summary(rec): - """One sentence describing what this account may reach. Deliberately says LESS than the - editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters - would be wrong the moment a filter got interesting.""" - import core.perm_scope as perm_scope - - if perms.is_admin(rec): - # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if - # they applied is the one misreading of that clause that could actually hurt. - return "Everything (admin)" - if not perm_scope.is_migrated(rec): - mods = perms.allowed_modules(rec) - return "All modules (legacy rules)" if mods is None else \ - f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" - entries = (rec.get("perms") or {}) - opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] - restricted = sum(1 for k in opened - if (entries[k].get("filter") or entries[k].get("hiddenFields"))) - if not opened: - return "No modules" - base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" - return f"{base}, {restricted} restricted" if restricted else base - - -# ── request validation: every rule below is fail-closed ────────────────────────────────────────── -def _clean_username(v): - uname = str(v or "").strip().lower() - if not _UNAME_RE.match(uname): - raise err(400, "bad_username", - "a username is 2-32 characters: lowercase letters, digits, dot, dash or " - "underscore, starting with a letter or digit") - return uname - - -def _clean_password(v): - pw = str(v or "") - if len(pw) < _MIN_PW: - raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") - return pw - - -def _clean_role(v): - role = str(v or "").strip().lower() - if role not in _ROLES: - raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") - return role - - -def _clean_bus(v): - """'all', or a non-empty list of KNOWN business-unit ids. - - Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no - recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_bus", - "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " - "would read as unrestricted)") - out = [] - for b in v: - try: - b = int(b) - except (TypeError, ValueError): - raise err(400, "bad_bus", "a business-unit id must be a number") - if b not in users.BU_LABELS: - raise err(400, "bad_bus", - f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " - "unrecognised id would widen access to every BU") - out.append(b) - return sorted(set(out)) - - -def _clean_modules(v): - """'all', or a non-empty list of KNOWN module keys. - - `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is - refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would - silently lock the account out of the page they meant to grant. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_modules", - "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " - "reads as UNRESTRICTED, which is the opposite of locking an account down") - known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) - out, bad = [], [] - for k in v: - k = str(k or "").strip() - (out if k in known else bad).append(k) - if bad: - raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") - return sorted(set(out)) - - -def _tenant_of(rec): - """The account's company, with the pre-wave default. ONE spelling of the default, used by - every admin route — two spellings is how a tenant wall grows a gap.""" - return str((rec or {}).get("tenant") or "royal-imports").strip().lower() - - -# ── the routes ─────────────────────────────────────────────────────────────────────────────────── -@router.get("/admin/users") -def list_users(session: Session = Depends(admin_gate)): - """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global - control-plane bucket, so without this filter a Nurilab admin would read Royal's whole - roster — names, emails, agents — from one URL.""" - reg = _registry() - mine = _tenant_of(session.user) - return {"users": [_view(u, reg[u]) for u in sorted(reg) - if _tenant_of(reg[u]) == mine]} - - -@router.post("/admin/users", status_code=201) -def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): - """Create an account. **409 if the username already exists — `PATCH` is the update path.** - - ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh - record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and - RESURRECT every cookie minted before its last password change. `core/users.py` now - preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / - update a user" calls the same function), but this route still refuses: an upsert that silently - replaces an account's password, role and BU access because someone reused a username is not an - update anybody asked for. - """ - body = body or {} - _require_store() - uname = _clean_username(body.get("username")) - pw = _clean_password(body.get("password")) - role = _clean_role(body.get("role") or "user") - bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") - modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") - name = str(body.get("name") or "").strip() or uname - agent = str(body.get("agent") or "").strip() or None - email = str(body.get("email") or "").strip() or None - - if uname in _registry(): - raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") - # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — - # the tenant is stamped from the session, not taken from the body. - users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, - agent=agent, email=email, tenant=_tenant_of(session.user)) - reg = _registry() - if uname not in reg: - # The store accepted the write and does not have it. Reporting 201 here would be the - # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. - raise err(503, "store_unavailable", "the account was not saved — try again") - return {"user": _view(uname, reg[uname])} - - -@router.patch("/admin/users/{username}") -def update_user(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Change access on an existing account. Only the keys PRESENT in the body change. - - ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every - request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very - next call — bumping the epoch would only add a forced re-login on top. `active: false` is the - exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its - session rather than keep working until the cookie expires. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if not body: - raise err(400, "empty_patch", "no fields to update") - - unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) - if unknown: - # A silently-ignored field is how a UI ends up believing it saved something it did not. - raise err(400, "unknown_fields", f"cannot update: {unknown}") - - is_self = uname == session.uname - role = _clean_role(body["role"]) if "role" in body else None - active = bool(body["active"]) if "active" in body else None - # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is - # not the thing that keeps the product reachable — it just stops an administrator removing - # their own access with a toggle and having to go find the master password. - if is_self and role is not None and role != "admin": - raise err(400, "self_demote", - "you cannot remove your own administrator role — ask another admin") - if is_self and active is False: - raise err(400, "self_deactivate", "you cannot deactivate your own account") - - kw = {} - if role is not None: - kw["role"] = role - if "name" in body: - kw["name"] = str(body["name"] or "").strip() or uname - if "bus" in body: - kw["bus"] = _clean_bus(body["bus"]) - if "modules" in body: - kw["modules"] = _clean_modules(body["modules"]) - if "agent" in body: - kw["agent"] = str(body["agent"] or "").strip() # '' clears the link - if "email" in body: - kw["email"] = str(body["email"] or "").strip() - if kw: - users.set_access(uname, **kw) - if active is not None and active != bool(reg[uname].get("active", True)): - users.set_active(uname, active) # bumps the epoch: kills live sessions - - fresh = _registry() - return {"user": _view(uname, fresh[uname])} - - -@router.post("/admin/users/{username}/password", status_code=204) -def set_password(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump - happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two - can never disagree. - - 204 with no body: there is nothing to say that the caller did not already know, and echoing the - account back would invite a client to diff a response for a password change. - """ - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - pw = _clean_password((body or {}).get("password")) - before = int(reg[uname].get("epoch") or 0) - users.set_password(uname, pw) - after = int((_registry().get(uname) or {}).get("epoch") or 0) - if after <= before: - # The whole value of this route is the revocation. If the epoch did not move, the sessions - # were not revoked, and answering 204 would say they were. - raise err(503, "store_unavailable", - "the password change did not persist — outstanding sessions were NOT revoked") - return Response(status_code=204) - - -# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── -#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema -#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list -#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` -#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), -#: so a migrated account would be locked out of Product with no control anywhere to grant it: -#: fail-closed, but unadministrable. The editor grows its section with zero client edits -#: (S3 built it off this route's `modules` + `fields_by_module`). -#: -#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be -#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple -#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid -#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` -#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, -#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` -#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s -#: `section_tenant_derived` NC does exactly that and reds. -#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from -#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, -#: non-group_only registry row that declares a `topic`. What stays hand-written is -#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` -#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this -#: editor at all. The intersection is the honest answer to "what can this module build pickers for". -#: -#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads -#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the -#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise -#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red -#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives -#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. -_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) - - -def _governable_catalogue(): - """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. - - ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is - tenant-blind by design. Two filters in two files answering one question is how the perms route - and `routes_nav` came apart in the first place. - """ - import core.registry as registry - return [{"key": v["key"], "label": v["label"]} - for v in registry.governable_modules().values() - if v.get("key") in _FIELD_PROVIDER_KEYS] - - -_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) - - -def _perm_modules(session): - """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. - - Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned - module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and - the tenant's own `ut_*` databases read through the DEFINITIONS projection. - - ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE - resolver for "which databases may this account see", it is `may_open`-filtered, and it reads - the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. - The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver - instead of assuming that is what keeps this from becoming the SECOND idea of who may see a - table (`user_tables.may_open`'s own docstring is the record of the first time that happened). - """ - import core.perms as _perms - import core.user_tables as user_tables - try: - ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, - st=session.runtime) - except Exception: - # A store that cannot list the tenant's databases must not take the whole editor down — - # the topic half is still administrable, and an empty `ut_*` half is visibly empty. - ut = () - return _perms.tenant_governable_modules(session.runtime, _governable_catalogue(), - ut_entries=ut) - - -def _enforced_keys(session): - """The module keys a permission block may actually be STORED against for this tenant.""" - import core.perms as _perms - return _perms.enforced_module_keys(_perm_modules(session)) - - -def _module_fields(key): - """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, - so an admin filters on exactly what the user sees. - - ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing - what to hide, so hiding the choices would make a field unhideable the moment it was hidden - once for somebody. The route is admin-gated; the projection a MEMBER receives is the one - `perm_scope.visible_fields` narrows. - - ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the - editor the customer schema for it would let an admin build a wall out of columns that - module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, - a filter naming a field the product rows lack would DENY every row via `permits()`. - The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader - is served fewer columns, but the ADMIN is choosing what may ever be hidden. - - ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read - `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that - was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being - a two-element literal, a `ut_*` database or a third topic would have been handed customer - columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the - stored wall would name columns that module does not have — which `permits()` resolves by - DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: - a topic without a schema arm is a missing arm, never a customer grid in disguise. - """ - import aios_grid - providers = {"customer_data": lambda: aios_grid.FIELDS, - "product_data": aios_grid.product_fields} - provider = providers.get(str(key or "")) - if provider is None: - # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative - # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is truthy. - raise err(400, "ungoverned_module", - f"{key!r} has no permission-editor field schema — the editor governs " - f"{sorted(providers)}") - src = provider() - known = _server_side_vocabularies() - out = [] - for f in src: - if not isinstance(f, dict) or not f.get("key"): - continue - opts = f.get("options") - if not isinstance(opts, list) or not opts: - # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See - # `_server_side_vocabularies`. - opts = known.get(f["key"]) - out.append({"key": f["key"], "label": f.get("label") or f["key"], - "type": f.get("type") or "text", - **({"options": list(opts)} if isinstance(opts, list) and opts else {}), - **({"pinned": True} if f.get("pinned") else {})}) - return out - - -def _server_side_vocabularies(): - """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, - but which the grid contract does not declare. - - ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid - resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the - field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so - every column that leans on the second branch renders `Select…` with nothing in it — the - identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix - belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied - list as authoritative *including when it is empty* ("this column has no values" is an answer), - which is correct and must not be weakened. So the caller stops handing it nothing. - - ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, - the same constant `_bucket()` mints from — a copy here would be a second vocabulary that - drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy - for the same reason `aios_grid`'s is: this route should not pull the analytics stack at - module load. - - ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than - let them look handled: - · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no - closed list to declare, and discovering it would mean a pool read on an admin route. - · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E - answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which - is the right home (the grid picks a declared list up for free) and is why this function - never needed an arm for it. Recorded because the ask, not a guess, is what settled it. - """ - try: - import modules.inventory as inventory - labels = list(inventory.COVERAGE_LABELS or ()) - except Exception: - # A missing analytics import must not take the permission editor down; the dropdown - # degrades to today's empty one rather than the page to a 500. - return {} - return {"stock_bucket": labels} if labels else {} - - -def _clean_perms(v, enforced_keys=None, listed_keys=None): - """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. - - ⭐ WAVE 33 (W33-T33) — `enforced_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It - used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 - flagged: an admin of a tenant provisioned for neither grid module could still store a wall - for both. It now comes from `_enforced_keys(session)`, i.e. from the tenant's own catalogue. - The default is the catalogue itself so a test or a future caller cannot silently widen it. - - ⛔ AND "LISTED" IS NOT "ENFORCED". `listed_keys` is every database the editor SHOWS, which - includes this tenant's `ut_*` ones. `perm_scope` is never consulted on those (visibility - there is `user_tables.may_open` — creator, admin, or a `core.shares` grant), so a wall stored - against one would be INERT: the editor would say DENY and the table routes would keep - serving. It is refused with a message that says WHICH of the two it is, because - "unknown module key" for a database the admin can see on screen is the wrong sentence. - - ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately - drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION - filter is the opposite situation — an admin types "agent is Tara", a leaf is silently - dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole - book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: - the filter is re-validated with `clean_filter_tree` and the result must come back with the - SAME leaf count it went in with. - - Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` - validates the NODE LIST alone and never sees the root conjunction, so the two are checked - separately and a bare node list is refused rather than silently read as `and`). - """ - import aios_grid - - if v is None: - return None - if not isinstance(v, dict): - raise err(400, "bad_perms", "perms must be an object keyed by module") - governed = set(enforced_keys) if enforced_keys is not None else set(_PERM_MODULES) - shown = set(listed_keys) if listed_keys is not None else governed - inert = sorted(set(v) & (shown - governed)) - if inert: - raise err(400, "unenforced_module", - f"{inert}: this tenant can SEE these databases, but a permission wall stored " - f"against them would never be applied — `routes_tables` gates a `ut_*` " - f"database on `user_tables.may_open` (creator, admin, or a share), not on " - f"`perm_scope`. Saved silently, it would read as a restriction that is not " - f"there. Share the database instead, or arm `perm_scope` over `ut_*` first.") - unknown = sorted(set(v) - shown) - if unknown: - raise err(400, "bad_perms", - f"unknown module keys: {unknown} — this tenant's permission editor governs " - f"{sorted(governed)}") - out = {} - for key, raw in v.items(): - if not isinstance(raw, dict): - raise err(400, "bad_perms", f"{key}: each entry must be an object") - valid_keys = {f["key"] for f in _module_fields(key)} - hidden = raw.get("hiddenFields") or [] - if not isinstance(hidden, list): - raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") - bad = sorted({str(h) for h in hidden} - valid_keys) - if bad: - # A hiddenFields entry naming nothing hides nothing — and reads as a restriction - # that is not there. - raise err(400, "bad_perms", - f"{key}: hiddenFields names unknown fields {bad}") - tree = raw.get("filter") - clean_tree = None - if tree not in (None, {}, []): - if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): - raise err(400, "bad_filter", - f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " - f"the root conjunction, and an 'or' wall read as 'and' restricts " - f"nothing it was meant to") - conj = tree.get("conj", "and") - if conj not in ("and", "or"): - raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") - nodes = tree["nodes"] - cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) - if _leaf_count(cleaned) != _leaf_count(nodes): - raise err(400, "bad_filter", - f"{key}: the filter contains conditions this module cannot evaluate " - f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " - f"are per-user and cannot be a permanent rule). Refused rather than " - f"saved with the bad conditions silently removed, which would store a " - f"weaker wall than the one on screen.") - clean_tree = {"conj": conj, "nodes": cleaned} - out[key] = {"access": bool(raw.get("access", True)), - "filter": clean_tree, - "hiddenFields": sorted({str(h) for h in hidden})} - _refuse_unshapeable_bu(out) - return out - - -def _refuse_unshapeable_bu(perms_out): - """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. - - Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes - that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which - is what `perm_migrate` emits and what the condition builder produces for "is any of". An - admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a - `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through - `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries - Fisch+Royal numbers. The row list looks right. The revenue is another BU's. - - That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just - shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with - nothing anomalous about them. - - So it is refused at the WRITE, where a person is present to fix it. The discriminator is - narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like - `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and - consolidated values are the correct answer for it, so it is not caught here (`derive` also - refuses it) and must not be. - """ - import core.perm_scope as perm_scope - - for key, e in perms_out.items(): - tree = e.get("filter") - if not tree or not _mentions_dba(tree.get("nodes")): - continue - probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, - "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} - team_id, _ = perm_scope.derive_pool_scope(probe, key) - if team_id is None and _confines_to_one_bu(tree): - raise err(400, "bu_condition_shape", - f"{key}: this filter restricts the business unit in a way the data layer " - f"cannot push into the query, so the rows would be correct while the " - f"revenue figures on them stayed consolidated across both units. Express " - f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " - f"or an 'any of' over the brands — and keep any other rules alongside it.") - - -def _guard_bu_widening(rec, cleaned, confirmed): - """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. - - `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the - first save the filter IS the whole wall and the admin owns it. But the first save also - CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted - rather than the one the server returned will, on its next save, PUT a payload with no BU - condition — and a whole-record replace deletes it. Measured, not theorised: the same client - payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both - business units. - - So a save that REMOVES the brand confinement is refused unless it says it means to. Widening - BU access is a legitimate thing for an administrator to do — it is just never a thing to do - by accident, and the difference between the two is one explicit flag. - - ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU - restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the - admin edits it). The right shape is "you may widen, say so". - """ - import core.perm_scope as perm_scope - - if confirmed: - return - stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} - for key, new_entry in cleaned.items(): - if not new_entry.get("access", True): - # Revoking the module entirely is the OPPOSITE of widening, and the filter on a - # denied entry governs nothing. Guarding it would make "lock this account out" - # require a confirm_widen flag, which reads as nonsense to whoever hits it. - continue - old = stored.get(key) - if not isinstance(old, dict): - continue # nothing stored yet: the fold already handled it - old_tree, new_tree = old.get("filter"), new_entry.get("filter") - if not old_tree or not _confines_to_one_bu(old_tree): - continue # was not confined -> this save cannot widen it - if new_tree and _confines_to_one_bu(new_tree): - continue # still confined -> not a widening - raise err(400, "bu_widening", - f"{key}: this would REMOVE the business-unit restriction and give the account " - f"both units. If that is intended, resend with confirm_widen: true. If it is " - f"not, reload the account's permissions first — saving a filter loaded before " - f"the last change drops the conditions it did not know about.") - # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees - # the connection to derive_pool_scope, which is what the removed condition was feeding. - _ = perm_scope - - -def _mentions_dba(nodes): - for n in nodes or (): - if not isinstance(n, dict): - continue - if isinstance(n.get("children"), list): - if _mentions_dba(n["children"]): - return True - elif n.get("colId") == "dba": - return True - return False - - -def _confines_to_one_bu(tree): - """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? - - ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard - refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a - high-balance Royal customer satisfies it — so consolidated values are the correct answer and - refusing it would be a false positive. My first attempt evaluated synthetic rows carrying - only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact - wall into a "confinement" it is not. - - So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the - reader satisfies everything else; can they still see both brands?" If yes, the wall does not - confine and the pool is correctly consolidated. If no, the brand structure is doing the - confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. - """ - def admits(node, brand): - if isinstance(node.get("children"), list): - kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] - if not kids: - return True - return any(kids) if node.get("conj") == "or" else all(kids) - if node.get("colId") != "dba": - return True # every other condition: assume satisfied - want = str(node.get("value") or "").strip().lower() - op = node.get("op") - if op == "eq": - return brand.lower() == want - if op == "neq": - return brand.lower() != want - if op == "contains": - return want in brand.lower() - if op == "doesNotContain": - return want not in brand.lower() - if op == "isEmpty": - return False # a probe brand is never blank - if op == "isNotEmpty": - return True - return True # an op that cannot judge a brand does not confine - - nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") - root = {"conj": conj, "children": nodes} - admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} - if not admitted: - return False # admits nothing at all — a different problem - # 'Both' belongs to either unit, so it cannot by itself widen the answer. - return not ({"Fisch", "Royal"} <= admitted) - - -def _leaf_count(nodes): - n = 0 - for node in nodes or (): - if isinstance(node, dict) and isinstance(node.get("children"), list): - n += _leaf_count(node["children"]) - else: - n += 1 - return n - - -def _fold_legacy_scope(rec, cleaned): - """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. - - Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` - fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write - whose filter carries no BU condition silently promotes a Royal-only account to BOTH business - units: the wall that used to come from `bus` is gone and nothing replaced it. - - Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check - passed, because the model was doing exactly what it was told. The account simply received - the other BU's customers on the wire. - - So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as - `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a - perms block. From then on the admin sees those leaves in the editor (GET returns the folded - filter) and can change them like any other condition — which is R1's whole point, and why - this is a one-time fold rather than a permanent floor AND-ed on every write. - - Idempotent by construction: it runs only while the record is un-migrated, and the write that - calls it is the write that migrates it. - - ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE - NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` - has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent - filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into - the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for - a `bus:[5]` account the moment an admin pressed Save. - - Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against - `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a - cross-topic leaf could come through, which is why it is the only place that needs the rule. - - ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the - BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU - is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). - The pushdown falls back to the record's own `bus` for exactly the modules whose field - vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves - ship together or the second one is a leak. - """ - import core.perm_migrate as perm_migrate - import core.perm_scope as perm_scope - - if perm_scope.is_migrated(rec): - return cleaned - legacy = perm_migrate.filter_for(rec) - if not legacy: - return cleaned # nothing to preserve: the account was unrestricted - out = {} - for key, e in cleaned.items(): - legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) - tree = e.get("filter") - if not legacy_here: - # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than - # storing a wall it will read as "deny every row". - out[key] = e - continue - if not tree or not tree.get("nodes"): - folded = legacy_here - else: - # AND the two roots together. The submitted tree keeps its own conjunction by being - # nested as a GROUP — flattening an `or` tree into an `and` root would turn the - # admin's "A or B" into "A and B", which is a different and much narrower wall. - folded = {"conj": "and", - "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), - "children": list(tree["nodes"])}]} - out[key] = dict(e, filter=folded) - return out - - -def _prune_to_module(tree, valid_keys): - """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. - - Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies - every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the - reason its own docstring gives (a silently-weakened wall reads as "Saved"). - - ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than - the admin's intent and half of an `and` group is a wider one; neither is the thing that was - written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps - the surviving conditions ones somebody actually declared. - """ - if not isinstance(tree, dict): - return None - - def keep(node): - if not isinstance(node, dict): - return None - kids = node.get("children") - if isinstance(kids, list): - surviving = [keep(k) for k in kids] - if not kids or any(s is None for s in surviving): - return None - return dict(node, children=surviving) - return node if node.get("colId") in valid_keys else None - - nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] - return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None - - -@router.get("/admin/users/{username}/perms") -def get_perms(username: str, session: Session = Depends(admin_gate)): - """This account's permission block plus the field schema the editor needs to render it.""" - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - import core.perm_scope as perm_scope - - rec = reg[uname] - stored = rec.get("perms") or {} - # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. - # The PUT is a whole-record replace over this same module list, so a key declared here and - # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The - # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the - # access its legacy grant currently gives rather than a guess in either direction. - # - # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything - # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a - # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product - # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` - # has applied since wave 18) and merges the tenant's own `ut_*` databases. - modules = _perm_modules(session) - enforced = [m["key"] for m in modules if m.get("enforced")] - perms_out = {} - for k in enforced: - e = stored.get(k) - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_access(rec, k)), "filter": None, "hiddenFields": []} - return {"username": uname, - "perms": perms_out, - # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a - # migrated record and LEGACY on an un-migrated one: opposite meanings for identical - # JSON, so the client cannot render honestly without knowing which world it is in. - "perms_v": int(rec.get("perms_v") or 0), - # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can - # say "Everything (admin)" instead of rendering stored rules that do not apply. - "is_admin": perms.is_admin(rec), - # ⭐ `enforced` rides every row. A `ut_*` database is LISTED because the owner asked to - # see every database, and declared UNENFORCED because `perm_scope` is not consulted on - # that path — so the editor can say which it is instead of painting a dead panel - # (wave 26 item 24: an empty answer that reads as an answer). The LABEL comes from the - # registry / the table's own definition; `_MODULE_LABELS` was a hardcoded copy of two - # registry labels and is deleted with this change. - "modules": modules, - # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav - # schema envelope; the editor needs the field list and nothing else, and one shape - # beats two readings of a sentence. - # ENFORCED MODULES ONLY, and that is the honest shape: a picker for a database whose - # wall is never applied is a control that lies. An unenforced row carries no entry - # here at all, so a client cannot mistake `[]` for "this database has no fields". - "fields_by_module": {k: _module_fields(k) for k in enforced}} - - -@router.put("/admin/users/{username}/perms") -def put_perms(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this account's permission block wholesale. - - WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot - express a deletion without a second vocabulary for it. - - ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: - `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the - target's LIVE session on its very next call. Bumping would only add a forced re-login on top - of a change that has already taken effect. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - - # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses - # perm_scope entirely, so this cannot brick them today — but it stops an administrator - # writing a wall for themselves that would bite the moment their role changed. - if uname == session.uname: - raise err(400, "self_perms", - "you cannot set permissions on your own account — ask another admin") - - _mods = _perm_modules(session) - cleaned = _clean_perms(body.get("perms"), - enforced_keys={m["key"] for m in _mods if m.get("enforced")}, - listed_keys={m["key"] for m in _mods}) or {} - cleaned = _fold_legacy_scope(reg[uname], cleaned) - _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) - users.set_access(uname, perms=cleaned) - fresh = _registry() - rec = fresh.get(uname) or {} - if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: - # The store took the write and did not record it. A 200 here would tell an administrator - # the wall is up when it is not — the one direction this must never fail in. - raise err(503, "store_unavailable", - "the permissions were not saved — the account is UNCHANGED") - return {"username": uname, "perms": rec.get("perms") or {}, - "perms_v": int(rec.get("perms_v") or 0)} - - -def _store_binding(session): - """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. - - ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 - rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset - repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path - that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only - tenant #0. Reporting the process default here would show "staging store, all fine" to exactly - the tenants that were not fine. - - ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it - could not describe the store is worse than one that says it does not know. - """ - try: - import core.data_binding as data_binding # noqa: PLC0415 - import core.store as store # noqa: PLC0415 - bound = getattr(session.runtime, "store_handle", None) - repo = getattr(bound, "repo", None) or store.REPO - return data_binding.describe(repo) - except Exception as e: # noqa: BLE001 - return {"deployment": "unknown", "production": False, "repo": "", - "writable": False, "refusal": f"could not resolve the store binding: {e}", - "localOverride": False, "allowlist": [], "observed": {}} - - -@router.get("/settings") -def settings(session: Session = Depends(require_session)): - """The session's OWN settings — any authenticated user, not admin-only. - - `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a - client may know about itself (never a hash, never the epoch). `scope` restates it as the two - things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others - would answer a question about our customer list. - """ - modules = perms.allowed_modules(session.user) - return { - # The build this Space is running. Behind the session on purpose — see `main.py::health`, - # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a - # Space secret at every deploy; absent means "somebody started this container by hand". - "version": os.environ.get("AIOS_VERSION") or "unknown", - "user": _public_user(session.user), - "scope": { - "bus": perms.allowed_bu_labels(session.user), - "team_id": perms.scope_team_id(session.user), - "agent": perms.scope_agent(session.user), - "modules": sorted(modules) if modules is not None else "all", - # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey - # pickers rather than to enforce — hidden fields are stripped from every data wire - # regardless, so a client that ignores this is narrowed anyway, never widened. - "perms": session.user.get("perms") or {}, - }, - "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", - session.tenant)}, - # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. - # - # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. - # D-160 established that a Space's environment cannot be read from outside, so "is staging - # pointed at production data?" is a question only the container can answer — and it went - # unanswered for the whole window in which two builds wrote tenant #0's real store. - # `verify_live` asserts this field after logging in, which is a check that survives a role - # swap in a way that reading a deploy log never did. - # - # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a - # dataset repo id are public names, but a health endpoint is the first URL a scanner finds - # and it may not describe the deployment. No customer data is here — only which store, and - # yes or no. - "store": _store_binding(session), - # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client - # does not paint a "Manage users" button that 403s. - "admin": perms.is_admin(session.user), - # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail - # paints its entry iff this is true, and `routes_platform_admin` refuses every request - # that is not, so a client that ignored this would gain exactly nothing. Derived from the - # ONE predicate rather than restated: a second copy of a two-condition wall is a second - # place for one of the conditions to go missing. - "platformAdmin": platform_admin.is_platform_admin(session.user), - } +"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). + +These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` +modals, over the SAME `core/users` account store — so both front-ends administer one set of +accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. + +ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit +dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record +is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. + +⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` +has no caller anywhere in the shipped app — module grants are only settable by editing the store +JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both +asserted in `verify_api.py` section A: + + * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin + clearing every checkbox to lock an account down would grant it everything. + * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to + `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose + whole point is strict isolation. + +Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in +`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the +WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both +read semantics are untouched — the seam is write-strict / read-unchanged. + +⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` +one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no +accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". +""" +import os +import re + +from fastapi import APIRouter, Body, Depends, Response + +import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag +import core.registry as registry +import core.store as store + +from deps import Session, err, perms, require_session, users +from routes_auth import _public_user + +router = APIRouter(prefix="/api/v1") + +#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under +#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no +#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. +_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") +#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog +#: enforces nothing; this does, and the two are allowed to differ because only one of them is +#: reachable from the internet. +_MIN_PW = 8 +_ROLES = ("user", "admin") + + +def admin_gate(session: Session = Depends(require_session)) -> Session: + """401 without a session, 403 without the admin role. Role, not a module grant — mirroring + `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + return session + + +def _require_store(): + """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" + try: + ok = store.available() + except Exception: + ok = False + if not ok: + raise err(503, "store_unavailable", + "the tenant store is unavailable — accounts cannot be read or changed") + + +def _registry(): + _require_store() + try: + reg = users.registry() or {} + except Exception: + raise err(503, "store_unavailable", "the tenant store is unavailable") + return reg + + +def _view(uname, rec, governed=None, st=None): + """What an administrator may see about an account. NEVER `salt` or `hash` — this function is + the only projection these routes use, so there is one place that can leak and it does not. + + ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both, + resolved once for the whole request; the single-record routes pass neither, because their + caller fetches the editor payload separately and `GET /perms` is the authority there. + """ + return {"username": uname, + "name": rec.get("name") or uname, + "role": rec.get("role", "user"), + "bus": rec.get("bus", "all"), + "bu_labels": perms.allowed_bu_labels(rec), + "modules": rec.get("modules", "all"), + "agent": rec.get("agent") or None, + "email": rec.get("email") or None, + "tenant": _tenant_of(rec), + "active": bool(rec.get("active", True)), + # The session-revocation handle. An admin needs it: it is the only visible evidence + # that "sign them out everywhere" actually happened. + "epoch": int(rec.get("epoch") or 0), + # S3 ask 4 — one sentence for the roster's Access column, so the list does not need + # a /perms round trip per row to fill one cell. Computed from the same record the + # editor will open, so the two cannot disagree. + "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st), + "perms_v": int(rec.get("perms_v") or 0)} + + +def _access_summary(rec, uname="", governed=None, st=None): + """One sentence describing what this account may reach. Deliberately says LESS than the + editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters + would be wrong the moment a filter got interesting. + + ⛔⛔ W36-T22 — IT MUST COUNT WHAT THE EDITOR WOULD SHOW, NOT WHAT IS STORED, AND THE TWO STOPPED + AGREEING THE MOMENT `get_perms` LEARNED TO DEFAULT FROM `may_read`. This counted STORED ENTRIES + only, which WAS the same answer before this wave: a migrated account had an entry for every + governable database, because the editor wrote one per topic on every save and no `ut_*` entry + could exist at all. Now a `ut_*` key legitimately has NO entry and is still OPEN, so the roster + would say *"1 module"* about an account the editor shows reaching twelve. ⛔ The roster is the + screen an administrator reads FIRST, and two screens disagreeing about one account is the + defect this wave exists to remove, not a rounding difference. + + ⚠ `governed`/`st` ARE PASSED IN RATHER THAN DERIVED HERE, and that is a cost decision. This + runs once PER ROW of the roster; asking `may_read` per (account x database) without a lend + would be a whole-document copy per question — D-213's shape, one route over. The caller + resolves the tenant's list and ONE lend for the whole request. A caller that passes neither + keeps the stored-entry reading, which is right for the single-record routes: their client + fetches `GET /perms` separately and that payload is the authority. + """ + import core.perm_scope as perm_scope + + if perms.is_admin(rec): + # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if + # they applied is the one misreading of that clause that could actually hurt. + return "Everything (admin)" + if not perm_scope.is_migrated(rec): + mods = perms.allowed_modules(rec) + return "All modules (legacy rules)" if mods is None else \ + f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" + entries = (rec.get("perms") or {}) + if governed: + principal = users._public(uname, rec) if uname else rec + opened = [k for k in governed if perm_scope.may_read(principal, k, st=st)] + else: + opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] + restricted = sum(1 for k in opened + if isinstance(entries.get(k), dict) + and (entries[k].get("filter") or entries[k].get("hiddenFields"))) + if not opened: + return "No modules" + base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" + return f"{base}, {restricted} restricted" if restricted else base + + +# ── request validation: every rule below is fail-closed ────────────────────────────────────────── +def _clean_username(v): + uname = str(v or "").strip().lower() + if not _UNAME_RE.match(uname): + raise err(400, "bad_username", + "a username is 2-32 characters: lowercase letters, digits, dot, dash or " + "underscore, starting with a letter or digit") + return uname + + +def _clean_password(v): + pw = str(v or "") + if len(pw) < _MIN_PW: + raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + return pw + + +def _clean_role(v): + role = str(v or "").strip().lower() + if role not in _ROLES: + raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") + return role + + +def _clean_bus(v): + """'all', or a non-empty list of KNOWN business-unit ids. + + Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no + recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_bus", + "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " + "would read as unrestricted)") + out = [] + for b in v: + try: + b = int(b) + except (TypeError, ValueError): + raise err(400, "bad_bus", "a business-unit id must be a number") + if b not in users.BU_LABELS: + raise err(400, "bad_bus", + f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " + "unrecognised id would widen access to every BU") + out.append(b) + return sorted(set(out)) + + +def _clean_modules(v): + """'all', or a non-empty list of KNOWN module keys. + + `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is + refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would + silently lock the account out of the page they meant to grant. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_modules", + "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " + "reads as UNRESTRICTED, which is the opposite of locking an account down") + known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) + out, bad = [], [] + for k in v: + k = str(k or "").strip() + (out if k in known else bad).append(k) + if bad: + raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") + return sorted(set(out)) + + +def _tenant_of(rec): + """The account's company, with the pre-wave default. ONE spelling of the default, used by + every admin route — two spellings is how a tenant wall grows a gap.""" + return str((rec or {}).get("tenant") or "royal-imports").strip().lower() + + +# ── the routes ─────────────────────────────────────────────────────────────────────────────────── +@router.get("/admin/users") +def list_users(session: Session = Depends(admin_gate)): + """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global + control-plane bucket, so without this filter a Nurilab admin would read Royal's whole + roster — names, emails, agents — from one URL.""" + reg = _registry() + mine = _tenant_of(session.user) + # ⭐ W36-T22 — the tenant's governable list and ONE lend, resolved once for the whole roster so + # `_access_summary` can agree with the editor without paying a document copy per question. + import core.user_tables as _ut + try: + governed = [m["key"] for m in _perm_modules(session)] + lent = _ut.lend_defs(session.runtime) + except Exception: # noqa: BLE001 + # A store that cannot list the tenant's databases must not take the roster down. The + # summary degrades to the stored-entry reading, which is what it always was. + governed, lent = None, None + return {"users": [_view(u, reg[u], governed=governed, st=lent) for u in sorted(reg) + if _tenant_of(reg[u]) == mine]} + + +@router.post("/admin/users", status_code=201) +def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create an account. **409 if the username already exists — `PATCH` is the update path.** + + ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh + record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and + RESURRECT every cookie minted before its last password change. `core/users.py` now + preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / + update a user" calls the same function), but this route still refuses: an upsert that silently + replaces an account's password, role and BU access because someone reused a username is not an + update anybody asked for. + """ + body = body or {} + _require_store() + uname = _clean_username(body.get("username")) + pw = _clean_password(body.get("password")) + role = _clean_role(body.get("role") or "user") + bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") + modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") + name = str(body.get("name") or "").strip() or uname + agent = str(body.get("agent") or "").strip() or None + email = str(body.get("email") or "").strip() or None + + if uname in _registry(): + raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — + # the tenant is stamped from the session, not taken from the body. + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email, tenant=_tenant_of(session.user)) + reg = _registry() + if uname not in reg: + # The store accepted the write and does not have it. Reporting 201 here would be the + # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. + raise err(503, "store_unavailable", "the account was not saved — try again") + return {"user": _view(uname, reg[uname])} + + +@router.patch("/admin/users/{username}") +def update_user(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Change access on an existing account. Only the keys PRESENT in the body change. + + ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every + request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very + next call — bumping the epoch would only add a forced re-login on top. `active: false` is the + exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its + session rather than keep working until the cookie expires. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if not body: + raise err(400, "empty_patch", "no fields to update") + + unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) + if unknown: + # A silently-ignored field is how a UI ends up believing it saved something it did not. + raise err(400, "unknown_fields", f"cannot update: {unknown}") + + is_self = uname == session.uname + role = _clean_role(body["role"]) if "role" in body else None + active = bool(body["active"]) if "active" in body else None + # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is + # not the thing that keeps the product reachable — it just stops an administrator removing + # their own access with a toggle and having to go find the master password. + if is_self and role is not None and role != "admin": + raise err(400, "self_demote", + "you cannot remove your own administrator role — ask another admin") + if is_self and active is False: + raise err(400, "self_deactivate", "you cannot deactivate your own account") + + kw = {} + if role is not None: + kw["role"] = role + if "name" in body: + kw["name"] = str(body["name"] or "").strip() or uname + if "bus" in body: + kw["bus"] = _clean_bus(body["bus"]) + if "modules" in body: + kw["modules"] = _clean_modules(body["modules"]) + if "agent" in body: + kw["agent"] = str(body["agent"] or "").strip() # '' clears the link + if "email" in body: + kw["email"] = str(body["email"] or "").strip() + if kw: + users.set_access(uname, **kw) + if active is not None and active != bool(reg[uname].get("active", True)): + users.set_active(uname, active) # bumps the epoch: kills live sessions + + fresh = _registry() + return {"user": _view(uname, fresh[uname])} + + +@router.post("/admin/users/{username}/password", status_code=204) +def set_password(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump + happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two + can never disagree. + + 204 with no body: there is nothing to say that the caller did not already know, and echoing the + account back would invite a client to diff a response for a password change. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + pw = _clean_password((body or {}).get("password")) + before = int(reg[uname].get("epoch") or 0) + users.set_password(uname, pw) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # The whole value of this route is the revocation. If the epoch did not move, the sessions + # were not revoked, and answering 204 would say they were. + raise err(503, "store_unavailable", + "the password change did not persist — outstanding sessions were NOT revoked") + return Response(status_code=204) + + +# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── +#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema +#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list +#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` +#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), +#: so a migrated account would be locked out of Product with no control anywhere to grant it: +#: fail-closed, but unadministrable. The editor grows its section with zero client edits +#: (S3 built it off this route's `modules` + `fields_by_module`). +#: +#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be +#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple +#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid +#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` +#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, +#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` +#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s +#: `section_tenant_derived` NC does exactly that and reds. +#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from +#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, +#: non-group_only registry row that declares a `topic`. What stays hand-written is +#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` +#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this +#: editor at all. The intersection is the honest answer to "what can this module build pickers for". +#: +#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads +#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the +#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise +#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red +#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives +#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. +_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) + + +def _governable_catalogue(): + """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. + + ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is + tenant-blind by design. Two filters in two files answering one question is how the perms route + and `routes_nav` came apart in the first place. + """ + import core.registry as registry + return [{"key": v["key"], "label": v["label"]} + for v in registry.governable_modules().values() + if v.get("key") in _FIELD_PROVIDER_KEYS] + + +_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) + + +def _perm_modules(session): + """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. + + Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned + module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and + the tenant's own `ut_*` databases read through the DEFINITIONS projection. + + ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE + resolver for "which databases may this account see", it is `may_open`-filtered, and it reads + the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. + The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver + instead of assuming that is what keeps this from becoming the SECOND idea of who may see a + table (`user_tables.may_open`'s own docstring is the record of the first time that happened). + """ + import core.perms as _perms + import core.user_tables as user_tables + try: + ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, + st=session.runtime) + except Exception: + # A store that cannot list the tenant's databases must not take the whole editor down — + # the topic half is still administrable, and an empty `ut_*` half is visibly empty. + ut = () + return _perms.tenant_governable_modules(session.runtime, _governable_catalogue(), + ut_entries=ut) + + +#: ⛔ `_enforced_keys` IS DELETED (W36-T22 / CONTRACT C2). It asked `perms.enforced_module_keys` +#: which subset of the listed databases a wall could actually be stored against, and after R6 the +#: answer is "all of them" — a question with one possible answer is not a question. Both it and the +#: `perms` helper behind it are gone rather than made to return everything, because a predicate +#: that ignores its argument is the always-true flag C2 deletes, one indirection along. +#: What callers use instead is `{m["key"] for m in _perm_modules(session)}` — the SAME list the +#: editor renders, so "shown" and "storable" cannot drift apart again. + + +def _module_fields(key, session=None): + """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, + so an admin filters on exactly what the user sees. + + ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing + what to hide, so hiding the choices would make a field unhideable the moment it was hidden + once for somebody. The route is admin-gated; the projection a MEMBER receives is the one + `perm_scope.visible_fields` narrows. + + ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the + editor the customer schema for it would let an admin build a wall out of columns that + module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, + a filter naming a field the product rows lack would DENY every row via `permits()`. + The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader + is served fewer columns, but the ADMIN is choosing what may ever be hidden. + + ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read + `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that + was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being + a two-element literal, a `ut_*` database or a third topic would have been handed customer + columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the + stored wall would name columns that module does not have — which `permits()` resolves by + DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: + a topic without a schema arm is a missing arm, never a customer grid in disguise. + + ⭐⭐ W36-T22 / R6 — AND `ut_*` NOW HAS AN ARM, WHICH IS THE HALF THAT MAKES THE TOGGLE REAL. + Owner item 11: *"how come the database is only toggleable for Odoo customers and Odoo + products … EVERY database should be able to be toggleable by admin."* A `ut_*` database's + field contract is its OWN stored definition, so the picker offers exactly the columns that + database has. ⛔ ONE SOURCE, NOT TWO: the picker (`get_perms.fields_by_module`) and the + validator (`_clean_perms`) both call this, so a column the picker offers can never be a + column the validator rejects. Getting that wrong is worse than an error message — + `permits()` DENIES on a predicate it cannot answer, so a mismatched wall is a + deny-everything trap wearing a saved-successfully toast. + + ⚠ `session` IS REQUIRED FOR A `ut_*` KEY and unused for a topic: a stored definition is + per-tenant data and there is no tenant-blind way to read one. A caller that omits it is + refused LOUDLY rather than handed another tenant's schema — the same rule the `else` + fall-through above was deleted for. + """ + import aios_grid + key = str(key or "") + if key.startswith("ut_"): + if session is None: + raise err(400, "ungoverned_module", + f"{key!r} is a tenant database, so its columns can only be read for a " + f"known tenant — this caller passed no session") + import core.user_tables as user_tables + defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime)) + if not defn: + raise err(400, "ungoverned_module", + f"{key!r} is not a database in this workspace") + src = defn.get("fields") or [] + else: + providers = {"customer_data": lambda: aios_grid.FIELDS, + "product_data": aios_grid.product_fields} + provider = providers.get(key) + if provider is None: + # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative + # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is + # truthy. + raise err(400, "ungoverned_module", + f"{key!r} has no permission-editor field schema. The editor governs " + f"{sorted(providers)} and this tenant's own databases") + src = provider() + known = _server_side_vocabularies() + out = [] + for f in src: + if not isinstance(f, dict) or not f.get("key"): + continue + opts = f.get("options") + if not isinstance(opts, list) or not opts: + # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See + # `_server_side_vocabularies`. + opts = known.get(f["key"]) + out.append({"key": f["key"], "label": f.get("label") or f["key"], + "type": f.get("type") or "text", + **({"options": list(opts)} if isinstance(opts, list) and opts else {}), + **({"pinned": True} if f.get("pinned") else {})}) + return out + + +def _server_side_vocabularies(): + """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, + but which the grid contract does not declare. + + ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid + resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the + field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so + every column that leans on the second branch renders `Select…` with nothing in it — the + identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix + belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied + list as authoritative *including when it is empty* ("this column has no values" is an answer), + which is correct and must not be weakened. So the caller stops handing it nothing. + + ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, + the same constant `_bucket()` mints from — a copy here would be a second vocabulary that + drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy + for the same reason `aios_grid`'s is: this route should not pull the analytics stack at + module load. + + ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than + let them look handled: + · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no + closed list to declare, and discovering it would mean a pool read on an admin route. + · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E + answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which + is the right home (the grid picks a declared list up for free) and is why this function + never needed an arm for it. Recorded because the ask, not a guess, is what settled it. + """ + try: + import modules.inventory as inventory + labels = list(inventory.COVERAGE_LABELS or ()) + except Exception: + # A missing analytics import must not take the permission editor down; the dropdown + # degrades to today's empty one rather than the page to a 500. + return {} + return {"stock_bucket": labels} if labels else {} + + +def _clean_perms(v, governed_keys=None, session=None): + """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. + + ⭐ WAVE 33 (W33-T33) — `governed_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It + used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 + flagged: an admin of a tenant provisioned for neither grid module could still store a wall + for both. It now comes from `_perm_modules(session)`, i.e. from the tenant's own catalogue. + The default is the topic catalogue itself so a test or a future caller cannot silently widen + it. + + ⭐⭐ W36-T22 / CONTRACT C2 — **THE `enforced`/`listed` SPLIT IS GONE, AND SO IS + `unenforced_module`.** This function took TWO key sets and refused anything in the gap + between them, with a message that ended *"Share the database instead, or arm `perm_scope` + over `ut_*` first."* W36-T21 armed it. There is no gap left: every database the editor lists + is a database whose wall `routes_tables` applies on every read, so a wall stored against one + is enforced by the same code that enforces `customer_data`'s. **Two sets collapse into one**, + which is the shape C2 asks for — not a second set that is always equal to the first. + + ⚠ `session` IS THREADED SO `_module_fields` CAN READ A `ut_*` SCHEMA. Without it the + validator would have no field contract for the databases this ticket just made governable, + and "no contract" resolves to "every `hiddenFields` entry is unknown" — a 400 on the admin's + own screen. + + ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately + drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION + filter is the opposite situation — an admin types "agent is Tara", a leaf is silently + dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole + book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: + the filter is re-validated with `clean_filter_tree` and the result must come back with the + SAME leaf count it went in with. + + Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` + validates the NODE LIST alone and never sees the root conjunction, so the two are checked + separately and a bare node list is refused rather than silently read as `and`). + """ + import aios_grid + + if v is None: + return None + if not isinstance(v, dict): + raise err(400, "bad_perms", "perms must be an object keyed by module") + governed = set(governed_keys) if governed_keys is not None else set(_PERM_MODULES) + unknown = sorted(set(v) - governed) + if unknown: + raise err(400, "bad_perms", + f"unknown module keys: {unknown} — this tenant's permission editor governs " + f"{sorted(governed)}") + out = {} + for key, raw in v.items(): + if not isinstance(raw, dict): + raise err(400, "bad_perms", f"{key}: each entry must be an object") + valid_keys = {f["key"] for f in _module_fields(key, session=session)} + hidden = raw.get("hiddenFields") or [] + if not isinstance(hidden, list): + raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") + bad = sorted({str(h) for h in hidden} - valid_keys) + if bad: + # A hiddenFields entry naming nothing hides nothing — and reads as a restriction + # that is not there. + raise err(400, "bad_perms", + f"{key}: hiddenFields names unknown fields {bad}") + tree = raw.get("filter") + clean_tree = None + if tree not in (None, {}, []): + if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): + raise err(400, "bad_filter", + f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " + f"the root conjunction, and an 'or' wall read as 'and' restricts " + f"nothing it was meant to") + conj = tree.get("conj", "and") + if conj not in ("and", "or"): + raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") + nodes = tree["nodes"] + cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) + if _leaf_count(cleaned) != _leaf_count(nodes): + raise err(400, "bad_filter", + f"{key}: the filter contains conditions this module cannot evaluate " + f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " + f"are per-user and cannot be a permanent rule). Refused rather than " + f"saved with the bad conditions silently removed, which would store a " + f"weaker wall than the one on screen.") + clean_tree = {"conj": conj, "nodes": cleaned} + out[key] = {"access": bool(raw.get("access", True)), + "filter": clean_tree, + "hiddenFields": sorted({str(h) for h in hidden})} + _refuse_unshapeable_bu(out) + return out + + +def _refuse_unshapeable_bu(perms_out): + """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. + + Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes + that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which + is what `perm_migrate` emits and what the condition builder produces for "is any of". An + admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a + `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through + `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries + Fisch+Royal numbers. The row list looks right. The revenue is another BU's. + + That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just + shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with + nothing anomalous about them. + + So it is refused at the WRITE, where a person is present to fix it. The discriminator is + narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like + `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and + consolidated values are the correct answer for it, so it is not caught here (`derive` also + refuses it) and must not be. + """ + import core.perm_scope as perm_scope + + for key, e in perms_out.items(): + tree = e.get("filter") + if not tree or not _mentions_dba(tree.get("nodes")): + continue + probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, + "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} + team_id, _ = perm_scope.derive_pool_scope(probe, key) + if team_id is None and _confines_to_one_bu(tree): + raise err(400, "bu_condition_shape", + f"{key}: this filter restricts the business unit in a way the data layer " + f"cannot push into the query, so the rows would be correct while the " + f"revenue figures on them stayed consolidated across both units. Express " + f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " + f"or an 'any of' over the brands — and keep any other rules alongside it.") + + +def _guard_bu_widening(rec, cleaned, confirmed): + """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. + + `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the + first save the filter IS the whole wall and the admin owns it. But the first save also + CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted + rather than the one the server returned will, on its next save, PUT a payload with no BU + condition — and a whole-record replace deletes it. Measured, not theorised: the same client + payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both + business units. + + So a save that REMOVES the brand confinement is refused unless it says it means to. Widening + BU access is a legitimate thing for an administrator to do — it is just never a thing to do + by accident, and the difference between the two is one explicit flag. + + ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU + restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the + admin edits it). The right shape is "you may widen, say so". + """ + import core.perm_scope as perm_scope + + if confirmed: + return + stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} + for key, new_entry in cleaned.items(): + if not new_entry.get("access", True): + # Revoking the module entirely is the OPPOSITE of widening, and the filter on a + # denied entry governs nothing. Guarding it would make "lock this account out" + # require a confirm_widen flag, which reads as nonsense to whoever hits it. + continue + old = stored.get(key) + if not isinstance(old, dict): + continue # nothing stored yet: the fold already handled it + old_tree, new_tree = old.get("filter"), new_entry.get("filter") + if not old_tree or not _confines_to_one_bu(old_tree): + continue # was not confined -> this save cannot widen it + if new_tree and _confines_to_one_bu(new_tree): + continue # still confined -> not a widening + raise err(400, "bu_widening", + f"{key}: this would REMOVE the business-unit restriction and give the account " + f"both units. If that is intended, resend with confirm_widen: true. If it is " + f"not, reload the account's permissions first — saving a filter loaded before " + f"the last change drops the conditions it did not know about.") + # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees + # the connection to derive_pool_scope, which is what the removed condition was feeding. + _ = perm_scope + + +def _mentions_dba(nodes): + for n in nodes or (): + if not isinstance(n, dict): + continue + if isinstance(n.get("children"), list): + if _mentions_dba(n["children"]): + return True + elif n.get("colId") == "dba": + return True + return False + + +def _confines_to_one_bu(tree): + """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? + + ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard + refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a + high-balance Royal customer satisfies it — so consolidated values are the correct answer and + refusing it would be a false positive. My first attempt evaluated synthetic rows carrying + only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact + wall into a "confinement" it is not. + + So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the + reader satisfies everything else; can they still see both brands?" If yes, the wall does not + confine and the pool is correctly consolidated. If no, the brand structure is doing the + confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. + """ + def admits(node, brand): + if isinstance(node.get("children"), list): + kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] + if not kids: + return True + return any(kids) if node.get("conj") == "or" else all(kids) + if node.get("colId") != "dba": + return True # every other condition: assume satisfied + want = str(node.get("value") or "").strip().lower() + op = node.get("op") + if op == "eq": + return brand.lower() == want + if op == "neq": + return brand.lower() != want + if op == "contains": + return want in brand.lower() + if op == "doesNotContain": + return want not in brand.lower() + if op == "isEmpty": + return False # a probe brand is never blank + if op == "isNotEmpty": + return True + return True # an op that cannot judge a brand does not confine + + nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") + root = {"conj": conj, "children": nodes} + admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} + if not admitted: + return False # admits nothing at all — a different problem + # 'Both' belongs to either unit, so it cannot by itself widen the answer. + return not ({"Fisch", "Royal"} <= admitted) + + +def _leaf_count(nodes): + n = 0 + for node in nodes or (): + if isinstance(node, dict) and isinstance(node.get("children"), list): + n += _leaf_count(node["children"]) + else: + n += 1 + return n + + +def _fold_legacy_scope(rec, cleaned): + """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. + + Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` + fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write + whose filter carries no BU condition silently promotes a Royal-only account to BOTH business + units: the wall that used to come from `bus` is gone and nothing replaced it. + + Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check + passed, because the model was doing exactly what it was told. The account simply received + the other BU's customers on the wire. + + So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as + `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a + perms block. From then on the admin sees those leaves in the editor (GET returns the folded + filter) and can change them like any other condition — which is R1's whole point, and why + this is a one-time fold rather than a permanent floor AND-ed on every write. + + Idempotent by construction: it runs only while the record is un-migrated, and the write that + calls it is the write that migrates it. + + ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE + NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` + has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent + filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into + the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for + a `bus:[5]` account the moment an admin pressed Save. + + Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against + `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a + cross-topic leaf could come through, which is why it is the only place that needs the rule. + + ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the + BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU + is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). + The pushdown falls back to the record's own `bus` for exactly the modules whose field + vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves + ship together or the second one is a leak. + """ + import core.perm_migrate as perm_migrate + import core.perm_scope as perm_scope + + if perm_scope.is_migrated(rec): + return cleaned + legacy = perm_migrate.filter_for(rec) + if not legacy: + return cleaned # nothing to preserve: the account was unrestricted + out = {} + for key, e in cleaned.items(): + legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) + tree = e.get("filter") + if not legacy_here: + # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than + # storing a wall it will read as "deny every row". + out[key] = e + continue + if not tree or not tree.get("nodes"): + folded = legacy_here + else: + # AND the two roots together. The submitted tree keeps its own conjunction by being + # nested as a GROUP — flattening an `or` tree into an `and` root would turn the + # admin's "A or B" into "A and B", which is a different and much narrower wall. + folded = {"conj": "and", + "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), + "children": list(tree["nodes"])}]} + out[key] = dict(e, filter=folded) + return out + + +def _prune_to_module(tree, valid_keys): + """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. + + Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies + every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the + reason its own docstring gives (a silently-weakened wall reads as "Saved"). + + ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than + the admin's intent and half of an `and` group is a wider one; neither is the thing that was + written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps + the surviving conditions ones somebody actually declared. + """ + if not isinstance(tree, dict): + return None + + def keep(node): + if not isinstance(node, dict): + return None + kids = node.get("children") + if isinstance(kids, list): + surviving = [keep(k) for k in kids] + if not kids or any(s is None for s in surviving): + return None + return dict(node, children=surviving) + return node if node.get("colId") in valid_keys else None + + nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] + return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None + + +@router.get("/admin/users/{username}/perms") +def get_perms(username: str, session: Session = Depends(admin_gate)): + """This account's permission block plus the field schema the editor needs to render it.""" + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + import core.perm_scope as perm_scope + + rec = reg[uname] + stored = rec.get("perms") or {} + # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. + # The PUT is a whole-record replace over this same module list, so a key declared here and + # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The + # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the + # access its legacy grant currently gives rather than a guess in either direction. + # + # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything + # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a + # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product + # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` + # has applied since wave 18) and merges the tenant's own `ut_*` databases. + modules = _perm_modules(session) + # ⭐⭐ W36-T22 / C2 — EVERY listed database, not an `enforced` subset of them. See + # `perms.tenant_governable_modules` for why the flag is deleted rather than defaulted. + governed = [m["key"] for m in modules] + perms_out = {} + for k in governed: + e = stored.get(k) + # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN + # OUTAGE. `may_access` reads migrated-and-undeclared as DENY. That is right for a registry + # topic and catastrophic for a `ut_*` key, because no `ut_*` entry was STORABLE before this + # wave — so every migrated account carries none, and this route would default every one of + # them to `access: false`. + # + # ⛔ AND IT WOULD NOT HAVE STAYED A DISPLAY BUG FOR LONG. C2 also deleted the filters that + # kept `ut_*` keys OUT of the PUT body, so the editor now sends every declared module: an + # administrator opening this page, changing one unrelated filter and pressing Save would + # write an EXPLICIT `{access: false}` for all ten keychain databases — at which point + # `may_read`'s deny-only overlay fires and revokes them for real. One click, silent, + # permanent. Defaulting from the SAME evaluator the read door uses is what makes the page + # show what the user can actually open, so a save of an untouched payload is a no-op. + # ⚠ `_public(uname, rec)`, NOT the raw record. A stored record is keyed BY username in + # `users.json` and does not carry one INSIDE it, so `may_read` would hand `may_open` a + # `None` viewer and get a fail-closed False — the very outage this line exists to prevent, + # arriving through the fix for it. The same trap cost a gate double an hour earlier today. + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(perm_scope.may_read(users._public(uname, rec), k, + st=session.runtime)), + "filter": None, "hiddenFields": []} + return {"username": uname, + "perms": perms_out, + # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a + # migrated record and LEGACY on an un-migrated one: opposite meanings for identical + # JSON, so the client cannot render honestly without knowing which world it is in. + "perms_v": int(rec.get("perms_v") or 0), + # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can + # say "Everything (admin)" instead of rendering stored rules that do not apply. + "is_admin": perms.is_admin(rec), + # ⭐⭐ W36-T22 / C2 — `{key, label}` per row and NO `enforced`. It used to ride here + # so the editor could paint an apology ("Not set here") for a database whose wall was + # never applied; W36-T21 armed every one of them, so the flag would now be constantly + # true and the apology would be a lie with a green gate behind it. The LABEL comes + # from the registry / the table's own definition. + "modules": modules, + # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav + # schema envelope; the editor needs the field list and nothing else, and one shape + # beats two readings of a sentence. + # ⭐ EVERY governed database now carries its fields, `ut_*` included — that is owner + # item 11's *"EVERY database should be able to be toggleable by admin"*, and it is + # the SAME resolver `_clean_perms` validates against, so the picker cannot offer a + # column the validator will reject. + "fields_by_module": {k: _module_fields(k, session=session) for k in governed}} + + +@router.put("/admin/users/{username}/perms") +def put_perms(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this account's permission block wholesale. + + WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot + express a deletion without a second vocabulary for it. + + ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: + `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the + target's LIVE session on its very next call. Bumping would only add a forced re-login on top + of a change that has already taken effect. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + + # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses + # perm_scope entirely, so this cannot brick them today — but it stops an administrator + # writing a wall for themselves that would bite the moment their role changed. + if uname == session.uname: + raise err(400, "self_perms", + "you cannot set permissions on your own account — ask another admin") + + _mods = _perm_modules(session) + # ⭐⭐ W36-T22 / C2 — ONE key set. It was two ("listed" and "enforced") with a refusal in the + # gap; W36-T21 closed the gap, so every database the editor shows is one whose wall the table + # routes apply. `session` rides so a `ut_*` schema can be read for THIS tenant. + cleaned = _clean_perms(body.get("perms"), + governed_keys={m["key"] for m in _mods}, session=session) or {} + cleaned = _fold_legacy_scope(reg[uname], cleaned) + _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) + users.set_access(uname, perms=cleaned) + fresh = _registry() + rec = fresh.get(uname) or {} + if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: + # The store took the write and did not record it. A 200 here would tell an administrator + # the wall is up when it is not — the one direction this must never fail in. + raise err(503, "store_unavailable", + "the permissions were not saved — the account is UNCHANGED") + return {"username": uname, "perms": rec.get("perms") or {}, + "perms_v": int(rec.get("perms_v") or 0)} + + +def _store_binding(session): + """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. + + ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 + rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset + repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path + that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only + tenant #0. Reporting the process default here would show "staging store, all fine" to exactly + the tenants that were not fine. + + ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it + could not describe the store is worse than one that says it does not know. + """ + try: + import core.data_binding as data_binding # noqa: PLC0415 + import core.store as store # noqa: PLC0415 + bound = getattr(session.runtime, "store_handle", None) + repo = getattr(bound, "repo", None) or store.REPO + return data_binding.describe(repo) + except Exception as e: # noqa: BLE001 + return {"deployment": "unknown", "production": False, "repo": "", + "writable": False, "refusal": f"could not resolve the store binding: {e}", + "localOverride": False, "allowlist": [], "observed": {}} + + +@router.get("/settings") +def settings(session: Session = Depends(require_session)): + """The session's OWN settings — any authenticated user, not admin-only. + + `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a + client may know about itself (never a hash, never the epoch). `scope` restates it as the two + things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others + would answer a question about our customer list. + """ + modules = perms.allowed_modules(session.user) + return { + # The build this Space is running. Behind the session on purpose — see `main.py::health`, + # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a + # Space secret at every deploy; absent means "somebody started this container by hand". + "version": os.environ.get("AIOS_VERSION") or "unknown", + "user": _public_user(session.user), + "scope": { + "bus": perms.allowed_bu_labels(session.user), + "team_id": perms.scope_team_id(session.user), + "agent": perms.scope_agent(session.user), + "modules": sorted(modules) if modules is not None else "all", + # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey + # pickers rather than to enforce — hidden fields are stripped from every data wire + # regardless, so a client that ignores this is narrowed anyway, never widened. + "perms": session.user.get("perms") or {}, + }, + "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", + session.tenant)}, + # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. + # + # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. + # D-160 established that a Space's environment cannot be read from outside, so "is staging + # pointed at production data?" is a question only the container can answer — and it went + # unanswered for the whole window in which two builds wrote tenant #0's real store. + # `verify_live` asserts this field after logging in, which is a check that survives a role + # swap in a way that reading a deploy log never did. + # + # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a + # dataset repo id are public names, but a health endpoint is the first URL a scanner finds + # and it may not describe the deployment. No customer data is here — only which store, and + # yes or no. + "store": _store_binding(session), + # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client + # does not paint a "Manage users" button that 403s. + "admin": perms.is_admin(session.user), + # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail + # paints its entry iff this is true, and `routes_platform_admin` refuses every request + # that is not, so a client that ignored this would gain exactly nothing. Derived from the + # ONE predicate rather than restated: a second copy of a two-condition wall is a second + # place for one of the conditions to go missing. + "platformAdmin": platform_admin.is_platform_admin(session.user), + } diff --git a/api/routes_agent_harness.py b/api/routes_agent_harness.py new file mode 100644 index 0000000000000000000000000000000000000000..cfe1ffa6958ea00035eaaaf64d5c826e246fb209 --- /dev/null +++ b/api/routes_agent_harness.py @@ -0,0 +1,455 @@ +"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. + +Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file +pertaining to the agent skills, router etc. So we build a user can build a custom harness for each +agent through the chat interface."* + + GET /api/v1/agents/{id}/harness the file LIST (no bodies) + GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions + GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim + PUT /api/v1/agents/{id}/harness write a NEW version of one file + DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history + +⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write +appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the +history is a record of what happened rather than of what somebody last wanted it to look like. + +⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, +`routes_automation`); a harness file is not. Do not copy this file's posture over there or that +one's over here — the two rulings differ on purpose and they sit one screen apart in the product. + +⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every +write through this router is made BY a signed-in administrator, so `author` is stamped from the +session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by +a person or by the agent in the chat — a claim the client is entitled to make, because both are +permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side +door the automation engine uses when the agent writes with no session at all; that one stamps the +agent's own id as the author, which is the only case where `author` is not a username. + +⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of +one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in +`trimmed` on every payload that mentions the file, so a reader can see that the history is partial +rather than infer that the file was only ever saved twice. This is the tenant document, which is +already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is +a store-sized leak with a UI in front of it. +""" +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule +#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. +HARNESS_KEY = "agent_harness" + +#: One body. Generous for a skill or a router file and far under the point where a single write +#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a +#: truncation: a silently truncated skill file is a harness that does not do what its text says. +MAX_BODY_BYTES = 128 * 1024 + +#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving +#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. +MAX_FILES = 64 + +#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). +MAX_VERSIONS = 100 + +MAX_PATH = 200 + +#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — +#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is +#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a +#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a +#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. +_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") + +AUTHOR_KINDS = ("user", "agent") + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must + degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" + try: + found = rt.get(HARNESS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _files(rt, agent_id): + found = _all(rt).get(str(agent_id)) + return found if isinstance(found, dict) else {} + + +#: What kind of principal an id names. `None` = this tenant has no agent with that id. +#: +#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY +#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under +#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — +#: `Shell.tsx` mounts ``, and +#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds +#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that +#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival +#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on +#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO +#: occurrences of Canvas, Properties, Run history or panelTabs. +#: +#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent +#: skills, router" files in two places is the parallel code path item 13 exists to refuse. +AGENT_SLACK = "slack" +AGENT_AUTOMATION = "automation" + + +def agent_kind(rt, agent_id): + """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. + + ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at + read time from a column definition or a connector schedule; they have no stored home, so a + harness file hung off one is an orphan the moment the column changes. `patch_automation` + already refuses those ids for the neighbouring reason, and this refuses them by simply not + finding them — `all_definitions` holds stored automations only. + """ + import routes_slack + aid = str(agent_id or "") + if isinstance(routes_slack._agents(rt).get(aid), dict): + return AGENT_SLACK + import automation_engine as engine + try: + known = engine.all_definitions(rt) or {} + except Exception: # noqa: BLE001 + return None + return AGENT_AUTOMATION if aid in known else None + + +def _known_agent(rt, agent_id): + """Does this tenant have an agent with this id, in EITHER registry?""" + return agent_kind(rt, agent_id) is not None + + +def agent_wall(session, agent_id): + """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. + + ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a + THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door + in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in + `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the + agent it hangs off, so it is reached by whoever may already configure that agent. Picking one + of the two walls for both would either lock the Agents module's own users out of a tab the + owner asked for, or hand the Slack permission wall to anyone with an automation grant. + """ + kind = agent_kind(session.runtime, agent_id) + if kind is None: + raise err(404, "no_agent", "there is no agent with that id in this workspace") + if kind == AGENT_SLACK: + import core.perms as perms + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + else: + session.require("automation") + return kind + + +def normalize_path(raw): + """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. + + ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There + are two ways into this store — the HTTP route (which must answer 400) and `record_version()` + (which must raise `ValueError`, having no response to put a status into). Written twice, the + two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one + copy is invisible because the other still refuses, so nothing goes red and the wall is now + half there. Written once, a change to the rule is felt at both doors and by the gate. + """ + path = str(raw or "").strip().strip("/") + if not path or len(path) > MAX_PATH: + return None + if set(path) - _PATH_OK or ".." in path or "//" in path: + return None + return path + + +def _clean_path(raw): + """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" + path = normalize_path(raw) + if path is None: + if not str(raw or "").strip().strip("/"): + raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") + if len(str(raw)) > MAX_PATH: + raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") + raise err(400, "bad_path", + "a harness path may use letters, digits, dot, dash, underscore and / only") + return path + + +def _blank(path, author, author_kind): + return {"path": path, "versions": [], "trimmed": 0, + "created": _now(), "createdBy": author, "createdKind": author_kind} + + +def _append(record, body, author, author_kind, restored_from=None): + """Append ONE version to a file record, in place, and report what was trimmed. + + The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. + A version list whose numbers restart at 1 after a trim would make two different bodies share a + name, and `restoredFrom` would then point at whichever one happened to be in the window. + """ + # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback + # the live document and may run it more than once; appending in place would then stack two + # copies of the same version into the history on a retry. + prior = record.get("versions") if isinstance(record.get("versions"), list) else [] + last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) + entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, + "created": _now(), "bytes": len(body.encode("utf-8"))} + if restored_from: + entry["restoredFrom"] = int(restored_from) + versions = [*prior, entry] + dropped = max(0, len(versions) - MAX_VERSIONS) + if dropped: + versions = versions[dropped:] + record["versions"] = versions + record["trimmed"] = int(record.get("trimmed") or 0) + dropped + return entry + + +def _head(record): + """The newest version of a file record, or `None` for a record with no versions at all.""" + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return versions[-1] if versions else None + + +def _row(record): + """One file, as the LIST door reports it: everything except the bodies. + + ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would + ship the whole harness on every pane render; the Harness tab lists first and opens one file + second, which is exactly the shape this answers. + """ + head = _head(record) or {} + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return {"path": record.get("path") or "", + "version": int(head.get("version") or 0), + "bytes": int(head.get("bytes") or 0), + "author": head.get("author") or record.get("createdBy") or "", + "authorKind": head.get("authorKind") or record.get("createdKind") or "user", + "updated": head.get("created") or record.get("created") or "", + "created": record.get("created") or "", + "versions": len(versions), + "trimmed": int(record.get("trimmed") or 0)} + + +def _version_rows(record): + """The history of one file, newest first, WITHOUT the bodies. + + A body per version is what makes a diff possible, and it is also what makes this payload big: + 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing + (`?path=…&version=N`), which is two round trips for a diff and none for a history list. + """ + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + out = [] + for entry in reversed(versions): + if not isinstance(entry, dict): + continue + row = {"version": int(entry.get("version") or 0), + "author": entry.get("author") or "", + "authorKind": entry.get("authorKind") or "user", + "created": entry.get("created") or "", + "bytes": int(entry.get("bytes") or 0)} + if entry.get("restoredFrom"): + row["restoredFrom"] = int(entry["restoredFrom"]) + out.append(row) + return out + + +def _limits(): + """The caps, IN the payload, so a client can say "this file is full" before a write fails. + + ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the + one cap that acts without refusing; these report the three that refuse. + """ + return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, + "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} + + +# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── +def record_version(runtime, agent_id, path, body, author="", author_kind="agent", + restored_from=None): + """Write one version from INSIDE the server — the agent's own half of R8. + + ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" + true. An agent acting inside an automation run holds no session and no cookie; if its only way + to write were the HTTP door it would have to borrow a person's identity, and the authorship + column would then be a record of who was logged in rather than of what wrote the file. + + Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a + `ValueError`, because a server-side caller has no HTTP response to put a 404 into. + """ + agent_id = str(agent_id or "") + if not _known_agent(runtime, agent_id): + raise ValueError(f"no agent {agent_id!r} in this workspace") + path = normalize_path(path) + if path is None: + raise ValueError("that is not an acceptable harness path") + body = str(body or "") + if len(body.encode("utf-8")) > MAX_BODY_BYTES: + raise ValueError("harness body is over the size limit") + kind = author_kind if author_kind in AUTHOR_KINDS else "agent" + author = str(author or agent_id) + # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a + # store `update` callback propagates out of a half-run read-modify-write, and the one thing a + # refusal must never do is leave the caller unsure whether the write happened. + existing = _files(runtime, agent_id) + if path not in existing and len(existing) >= MAX_FILES: + raise ValueError(f"this agent already has {MAX_FILES} harness files") + appended = {} + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) + appended.clear() + appended.update(_append(record, body, author, kind, restored_from)) + files[path] = record + cur[agent_id] = files + return cur + + runtime.update(HARNESS_KEY, _set, flush="sync") + return appended + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/agents/{agent_id}/harness") +def get_harness(agent_id: str, path: str = "", version: int = 0, + session: Session = Depends(require_session)): + """The list, one file, or one older body — decided by the query string (C4). + + ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than + consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to + editing a permission than to editing a document. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + files = _files(session.runtime, agent_id) + if not path: + rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] + return {"agent": agent_id, "files": rows, "limits": _limits()} + + wanted = _clean_path(path) + record = files.get(wanted) + if not isinstance(record, dict): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + if version: + for entry in versions: + if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): + return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", + "atVersion": int(version), "history": _version_rows(record), + "limits": _limits()} + # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked + # for something that existed and no longer does, and telling it apart from a bad number is + # the difference between "roll back to v3" failing loudly and failing as if v3 never was. + trimmed = int(record.get("trimmed") or 0) + if trimmed and int(version) <= trimmed: + raise err(410, "version_trimmed", + f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " + f"for this file, and its body is gone") + raise err(404, "no_version", f"this file has no version {int(version)}") + head = _head(record) or {} + return {"agent": agent_id, **_row(record), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(record), + "limits": _limits()} + + +@router.put("/agents/{agent_id}/harness") +def put_harness(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). + + ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a + UI affordance; it is this function refusing to have a code path that replaces a body. A + roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that + somebody went back rather than pretending the intervening versions never happened. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + body = body if isinstance(body, dict) else {} + path = _clean_path(body.get("path")) + text = body.get("body") + if not isinstance(text, str): + raise err(400, "no_body", "a harness file needs a body, even an empty one") + if len(text.encode("utf-8")) > MAX_BODY_BYTES: + raise err(413, "body_too_long", + f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") + + # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a + # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the + # session and is never read off the request. Nothing is bought by forging the first — both + # principals may write (R8) — and everything would be bought by forging the second. + kind = str(body.get("authorKind") or "user").strip().lower() + if kind not in AUTHOR_KINDS: + raise err(400, "bad_author_kind", "authorKind is either user or agent") + restored = body.get("restoredFrom") + try: + restored = int(restored) if restored else None + except (TypeError, ValueError): + raise err(400, "bad_version", "restoredFrom must be a version number") + + files = _files(session.runtime, agent_id) + if path not in files and len(files) >= MAX_FILES: + raise err(409, "too_many_files", + f"this agent already has {MAX_FILES} harness files; delete one to add another") + try: + record_version(session.runtime, agent_id, path, text, + author=session.uname, author_kind=kind, restored_from=restored) + except ValueError as exc: + raise err(400, "refused", str(exc)) + + fresh = _files(session.runtime, agent_id).get(path) + if not isinstance(fresh, dict) or not _head(fresh): + # The store took the write and did not record it. A 200 here would tell an administrator + # their skill file was saved when it was not — the shape `routes_slack` refuses too. + raise err(503, "store_unavailable", "the harness file was NOT saved") + head = _head(fresh) + return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), + "limits": _limits()} + + +@router.delete("/agents/{agent_id}/harness") +def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): + """Drop one harness file AND its history. + + ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing + versions as a side effect of an edit. Deleting a file is a person deciding the file should not + exist, which is a different act with a different button, and a store with no way to remove a + file is one where a typo'd path is permanent. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + wanted = _clean_path(path) + if wanted not in _files(session.runtime, agent_id): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + files.pop(wanted, None) + # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a + # bucket accumulates a row for every agent anybody ever opened the tab on. + if files: + cur[agent_id] = files + else: + cur.pop(agent_id, None) + return cur + + session.runtime.update(HARNESS_KEY, _set, flush="sync") + return {"agent": agent_id, "deleted": wanted, + "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) + if isinstance(rec, dict)], + "limits": _limits()} diff --git a/api/routes_alerts.py b/api/routes_alerts.py index 3ff47062772bd99f08e4cd0eb40d25e5e9382404..c651d88cf37431431a9588f38c540b71c89597df 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -1,668 +1,668 @@ -"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). - - GET /api/v1/alerts -> {alerts:[...]} - POST /api/v1/alerts <- {viewId, topic, label?} - DELETE /api/v1/alerts/{alert_id} - POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) - GET /api/v1/notifications -> {unread, items:[...]} - POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} - -The semantics — an alert is a view plus a remembered matched set, a notification is a NEW -ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. -This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. - -⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool -for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a -full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole -book, and the notification would name customers that user may not see — a permission leak wearing -a notification's clothes. The owner's own scope is the only correct basis for their alert. - -⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same -`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an -alert can see is by construction a row its owner could open. Re-implementing the filter here -would be a second definition of "matches", and those two would drift. -""" -import re - -from fastapi import APIRouter, Body, Depends - -import core.alerts as alerts -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. -_TOPICS = ("customer", "product") - -# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── -# -# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). -# -# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three -# keys onto the record at write time would give them to notifications minted AFTER the deploy and -# to nothing else — every notification already sitting in every tenant's inbox would open nothing, -# and the feature would be correct in the source and absent from the product -# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a -# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is -# another lane's file this wave — but that is the convenience, not the reason. -# -# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them -# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets -# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding -# here means the client branches on ONE field instead of re-deriving the same split. -# -# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` -# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by -# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) -# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, -# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review -# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real -# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored -# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary -# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed -# at was deleted two waves ago. - -#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them -#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). -NOTIF_KIND_ALERT = "alert" -NOTIF_KIND_AUTOMATION = "automation" -NOTIF_KIND_SHARE = "share" - -#: C3's `target.module` vocabulary, and the automation sub-selection. -TARGET_MODULE_DATABASE = "database" -TARGET_MODULE_AUTOMATION = "automation" -TARGET_TAB_RUNS = "runs" - -#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with -#: no producer is a string that reads as a feature — the reason this constant is named here and -#: cited from `routes_shares` rather than typed twice). -SHARE_TOPIC = "share" - -#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. -AUTOMATION_TOPIC = "automation" - -_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") - - -def route_for_topic(topic): - """A grid SCOPE key -> the registry route that renders it, or None. - - ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED - (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only - pair that differ — the registry names the surface (`customer_data`) while the grid names the - scope (`customer`) — so a topic passed through as a route sends every click to a page that - does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT - rather than plausible, because an absent target renders as a row that does not pretend to be - clickable, and a wrong one renders as a click that silently goes nowhere. - """ - t = str(topic or "").strip() - if t == "customer": - return "customer_data" - if t == "product": - return "product_data" - if _UT_TOPIC.match(t): - return t - return None - - -def _refusal_code(exc): - """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. - - ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` - (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own - cause. Anything that reduces all four to one word is throwing away the only information the - reader could have acted on. Returns `""` for a plain exception, so a caller can tell - "refused, and here is why" apart from "broke, and we do not know why". - """ - detail = getattr(exc, "detail", None) - if isinstance(detail, dict): - inner = detail.get("error") - if isinstance(inner, dict): - return str(inner.get("code") or "") - return "" - - -def notification_view(item): - """One STORED notification -> the shape the Inbox renders. PURE, and total. - - Never raises and never drops a row: an item it cannot classify comes back as an `alert` with - no `target`, which the client renders as an unclickable row rather than hiding. An inbox that - silently omits what it does not understand is the one failure a reader cannot detect. - """ - if not isinstance(item, dict): - return item - topic = str(item.get("topic") or "").strip() - alert_id = str(item.get("alertId") or "").strip() - - # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic - # says `automation` but whose producer key never arrived (a truncated payload, a server - # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work - # and silently does not, which is this repo's most-repeated failure shape. Failing the test - # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. - if topic == AUTOMATION_TOPIC and alert_id: - kind = NOTIF_KIND_AUTOMATION - target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} - elif topic == SHARE_TOPIC and alert_id: - # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, - # `row_id=`. - # - # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it - # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the - # target read `{module: "database", id: "view_42"}` — an instruction to open a database - # called `view_42`. It looked right in the payload and would have opened nothing. The - # producer resolves the object to its topic and hands over the route; this branch only - # shapes what it is given. - kind = NOTIF_KIND_SHARE - row_id = str(item.get("rowId") or "").strip() - target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, - **({"tab": row_id} if row_id else {})} - else: - kind = NOTIF_KIND_ALERT - route = route_for_topic(topic) - view_id = str(item.get("viewId") or "").strip() - target = None if route is None else ( - {"module": TARGET_MODULE_DATABASE, "id": route, - **({"tab": view_id} if view_id else {})}) - - # The email split: `subject` is the HEADER (what this is about — the alert, the automation, - # the database), `label` stays the BODY (what happened — the record that entered, the run - # summary). They were one field, which is why a notification read as a sentence with no - # sender and the pane could not be laid out like mail. - subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() - # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it - # is D-101's dead one; leaving it through would give the client two vocabularies for one - # question, which is the defect this wave's item 6 is about in a different file. - # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. - # - # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was - # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — - # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in - # the model or in the markup. Mail has a from. This is it. - # - # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. - # An alert firing and an automation landing rows have no person behind them; their honest - # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a - # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and - # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with - # you") — a sender recovered by regexing a sentence breaks the first time the sentence is - # reworded, and it would break silently, in the header. - # - # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an - # empty from column reads as a broken inbox rather than as an old notification. - actor = str(item.get("actor") or "").strip() - if kind == NOTIF_KIND_SHARE: - sender = actor or "A teammate" - elif kind == NOTIF_KIND_AUTOMATION: - # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's - # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` - # runs FIRST, so this server literal won and every actor-less automation notification - # showed the retired module name in the inbox's From column. - sender = actor or "Agents" - else: - sender = actor or "Alerts" - out = {**item, "read": bool(item.get("read")), "kind": kind, - "subject": subject or "Notification", "sender": sender} - if target is not None: - out["target"] = target - return out - - -def inbox_view(box): - """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. - - ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a - recount here would make the badge a function of whatever this page happened to include, which - is the exact defect `alertsModel.parseInbox`'s own header records from the other side. - """ - if not isinstance(box, dict): - return box - items = box.get("items") - if not isinstance(items, list): - return box - # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client - # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. - # - # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and - # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH - # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" - # against a browser clock would re-introduce the drift the offset exists to remove — a reader a - # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands - # now come from the same machine. - # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — - # the note two lines up records what happened last time only one of them was enriched. - return {**box, "now": alerts._now_iso(), - "items": [notification_view(n) for n in items]} - - -def _view_by_id(g, view_id): - """One saved view out of an assembly, by id. `None` when there is no such view. - - ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS - FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND - `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes - it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on - it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the - `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** - - ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In - `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare - FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our - side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above - the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical - line is swallowed by `/notifications`' `except Exception: continue`, so **every stored - view-alert was silently dropped from the Inbox** and nobody had anything to report at all. - One expression, one loud symptom and one silent one. - - ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site - be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. - - ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was - keyed `{id: view}` — which is precisely why the gate was green while production raised on - every call. The fixture is moving to the production shape in this same change, and tolerating - both here means a caller that legitimately holds one cannot resurrect the bug. - """ - want = str(view_id or "") - if not want: - return None - views = (g or {}).get("views") - if isinstance(views, dict): - found = views.get(want) - return found if isinstance(found, dict) else None - if not isinstance(views, list): - return None - for v in views: - if isinstance(v, dict) and str(v.get("id") or "") == want: - return v - return None - - -def _topic_or_400(raw): - topic = str(raw or "").strip().lower() - if topic.startswith("ut_") or topic in _TOPICS: - return topic - raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") - - -def _owner_session(session: Session, owner: str): - """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). - - ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an - owner session is built by swapping the `user` RECORD and letting both derive themselves. An - earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the - first write hook of the wave; the properties are the single definition of who a session is, - and going around them is how a session with an admin flag and a non-admin record exists. - - Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather - than evaluating as somebody else, which is the fail-closed direction. - """ - import core.users as users - - if str(owner) == str(session.uname): - return session - rec = (users.registry() or {}).get(str(owner)) - if not isinstance(rec, dict) or not rec.get("active", True): - return None - # `_public` is THE definition of what a session may know about its own account (never a hash - # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a - # second definition, and the one that leaks is always the copy. - return Session(tenant=session.tenant, user=users._public(str(owner), rec), - claims=session.claims, runtime=session.runtime) - - -def _evaluate(session: Session, rec: dict, assemblies=None): - """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. - - ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole - of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each - one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two - alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, - because each `_evaluate` was a closed call. - ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module - note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak - wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and - must never share an entry. Getting that key wrong is the one way this optimisation could leak. - ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: - they evaluate ONE alert and a memo for a single call is pure overhead. - """ - import aios_grid - from harness import filter_eval - - owner_sess = _owner_session(session, rec.get("owner")) - if owner_sess is None: - return {"skipped": "owner_unavailable"} - topic = str(rec.get("topic") or "") - memo_key = (topic, str(owner_sess.uname)) - g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None - if g is None: - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. - # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the - # one-shot field-name correction acks for every `ut_` topic that has an alert — - # taking them from the `/workspace` refresh that exists to show them to the person - # who made the edit. The customer branch below has always passed False; this one - # inherited a default nobody re-read. An inbox poll must never consume a one-shot. - g = ut_assembly(owner_sess, topic, - storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", - consume_corrections=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) - except Exception as e: # noqa: BLE001 - # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not - # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it - # keeps a blanket catch — but it now reports the refusal's OWN code where there is - # one. `type(e).__name__` said `HTTPException` for four different causes, and - # `lastError` is the only place a user ever learns why an alert stopped firing. - # - # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an - # evaluation genuinely needs the rows to run the filter over. So an alert on a - # read-through grid is created (T22) and then skips at evaluation with - # `window_required` naming why — which is D-184's remaining half, and it is a - # SENTENCE now rather than silence. - return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} - if isinstance(assemblies, dict): - assemblies[memo_key] = g - - view = _view_by_id(g, rec.get("viewId")) - if not isinstance(view, dict): - # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than - # deleting the alert: an alert that silently vanishes is indistinguishable from one that - # never fires, and the user cannot debug what is not there. - return {"skipped": "view_missing"} - - # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived - # and overlay values on a row. Evaluating a filter against raw pool dicts would silently - # never match any condition on a user-created or measure column. - # - # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND - # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. - # - # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows - # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner - # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: - # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell - # evaluated as BLANK. Measured on one assembly, same view, same rows: - # rows_src state='unpaid' / 'paid' - # _evaluate saw state='' / '' ⇐ every base cell blank - # the GRID saw state='unpaid' / 'paid' - # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` - # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it - # inverted.** End to end: a row whose value arrived by import, automation, paste or the create - # door never fired; only a value typed as a hand EDIT did. - # - # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; - # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); - # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns - # `skipped: window_required`). - # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it - # would let a stale definition value shadow an edit the user has just made — the same defect - # `table_rows`' own note records, arriving from the other side. - _ov = (g.get("ws") or {}).get("overlays") or {} - _merged = {} - for _r in g["rows_src"]: - _pid = str(_r.get("pid")) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _o = _ov.get(_pid) - if isinstance(_o, dict): - _cells.update(_o) - _merged[_pid] = _cells - rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, - derived=g.get("derived")) - config = view.get("config") or view - ctx = filter_eval.EvalCtx( - cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} - for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, - measure_sets=g.get("measure_sets") or {}, - today=g.get("today")) - pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, - member_pids=config.get("memberPids")) - labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} - return alerts.evaluate(rec.get("id"), [str(p) for p in pids], - labels=labels, partial=False, st=session.runtime) - - -@router.get("/alerts") -def list_alerts(session: Session = Depends(require_session)): - return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime)} - - -@router.post("/alerts") -def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): - body = body or {} - view_id = str(body.get("viewId") or "").strip() - if not view_id: - raise err(400, "bad_view", "an alert needs the id of the view it watches") - topic = _topic_or_400(body.get("topic")) - _require_filtered_view(session, topic, view_id) - import uuid - aid = f"al_{uuid.uuid4().hex[:12]}" - rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, - label=body.get("label") or "", st=session.runtime) - # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". - # Deferring this to the first write hook would mean the next edit announces the whole view. - outcome = _evaluate(session, rec) - return {"alert": {**rec, "seeded": True}, "first": outcome} - - -def _require_filtered_view(session: Session, topic: str, view_id: str): - """400 unless `view_id` exists on `topic` AND actually narrows something. - - ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one - that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" - (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an - entrant again — there is nothing left to enter. The owner's words are *"when a Record gets - into that Filter's criteria"*: no criteria, no alert, and said at creation rather than - discovered by never being notified. - - `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a - half-typed rule is not a filter, and this must agree with what actually narrows or it would - accept a view whose one rule the engine then ignores. - - ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and - the second one hid the first. - - (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is - `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a - read-through grid built the whole pool — and `scoped_pool` refuses that with - `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` - (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, - runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, - closed** — an alert on a read-through grid can now be made at all. - (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` - is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and - `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by - *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED - A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design - (a 5xx body is the server's internals), substituting *"Something went wrong on our - side."* — which is the owner's screenshot, word for word. A knowable cause returned as a - 5xx is invisible by construction, so re-wording the 503 could never have fixed this. - ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is - honest. What it may no longer do is catch a refusal that already knows its own name. - """ - from fastapi import HTTPException - - from harness import filter_eval - - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⚠ `consume_corrections=False` — the customer branch has always passed it and this - # one inherited a default nobody re-read. Creating an alert must not eat the one-shot - # field-name correction acks belonging to the `/workspace` refresh that exists to show - # them to the person who made the edit. Same defect `_evaluate`'s header records. - g = ut_assembly(session, topic, - storage_key=f"{session.tenant}:{topic}:{session.uname}", - consume_corrections=False, with_rows=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(session, scope=topic, consume_corrections=False) - except HTTPException: - raise # it already names its own cause - except Exception as e: # noqa: BLE001 - # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, - # the one path that reaches this branch is also the one path with nothing to debug from. - raise err(503, "unavailable", - f"the table could not be read ({type(e).__name__}) — try again in a moment") - view = _view_by_id(g, view_id) - if not isinstance(view, dict): - raise err(404, "no_view", "that view does not exist on this table") - nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) - - # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT - # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS - # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have - # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines - # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the - # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. - # - # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the - # fix was right, and the three reasons it hid are the most transferable thing in this file. - # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each - # naming themselves as the origin of the same screenshot are mutually exclusive, and the next - # reader believes whichever they meet first — which is why this correction sits above rather - # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead - # of the working tree. [[grep-output-is-not-source]] - # - # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, - # that this was owner item 17 — see the correction directly above). - # - # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other - # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF - # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. - # - # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the - # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. - # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever - # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that - # worked was the refusal path: "Alert me about new records" had never once created an alert - # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the - # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went - # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. - # - # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check - # arity until the line RUNS, and this line runs only on the success path of a feature whose - # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, - # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal - # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, - # thorough and honest about everything except the one path the feature exists for. - # - # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a - # column up in; building a second dict here would be a second answer to one question, which - # is this wave's other headline defect in a different file. Its leading underscore is a real - # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. - columns = filter_eval._columns_map(g.get("fields") or []) - - def _any_active(ns): - for n in ns or (): - if isinstance(n, dict) and isinstance(n.get("children"), list): - if _any_active(n["children"]): - return True - elif filter_eval.is_rule_active(n, columns): - return True - return False - - if not _any_active(nodes): - raise err(400, "no_filter", - "this view has no active filter, so no record can ever ENTER it — add a " - "condition to the view first, then create the alert") - - -@router.delete("/alerts/{alert_id}") -def delete_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - if str(rec.get("owner")) != str(session.uname) and not session.admin: - raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") - alerts.delete(alert_id, st=session.runtime) - return {"ok": True} - - -@router.post("/alerts/{alert_id}/run") -def run_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - return _evaluate(session, rec) - - -@router.get("/notifications") -def notifications(session: Session = Depends(require_session)): - """The inbox — RE-EVALUATED on read, which is a deliberate design choice. - - ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The - plan was a push hook: the automation engine calls `after_write` when it lands rows. But - `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be - evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside - a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity - construction that leaks scope. - - Pulling on read has none of that: the caller IS a session, the assemblies are already - scope-cached, and the user cannot observe the difference — an inbox is only ever read by - someone opening it. The cost is that a notification is minted when you LOOK rather than when - the row landed, so the `at` stamp is detection time, not arrival time. - - `after_write` stays exported for the day the engine can hand over a real identity. - - ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. - ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is - **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** - (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the - re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the - body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole - grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which - is the difference between "fine" and "three seconds per alert" the day somebody uses the - feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what - this does: every alert is still evaluated, against the same rows, in the same order. - """ - assemblies = {} - for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): - try: - _evaluate(session, rec, assemblies=assemblies) - except Exception: # noqa: BLE001 - continue # one bad alert must not empty the pane - # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before - # this wave carries a `target` too. See `notification_view`'s header for why it is derived. - return inbox_view(alerts.inbox(session.uname, st=session.runtime)) - - -@router.post("/notifications/read") -def read_notifications(body: dict = Body(default=None), - session: Session = Depends(require_session)): - body = body or {} - ids = body.get("ids") - if ids is not None and not isinstance(ids, list): - raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") - # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox - # module re-renders from it — an un-enriched answer here would strip `target` off every row - # the moment somebody marked one read, i.e. the feature would work until first use. - return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), - st=session.runtime)) - - -def after_write(session: Session, topic_key: str): - """THE WRITE HOOK — call after a write that could change what a view matches. - - Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation - upserts reach it the same way. It never raises: an alert evaluation failing must not fail the - edit that triggered it. - - ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes - one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. - ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the - push hook was resolved the other way). Booked rather than wired: minting a session inside the - engine's worker thread is the ad-hoc identity construction this file exists to avoid. - """ - try: - assemblies = {} - return alerts.after_write(topic_key, st=session.runtime, - runner=lambda rec: _evaluate(session, rec, - assemblies=assemblies)) - except Exception: # noqa: BLE001 - return {"evaluated": 0} +"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). + + GET /api/v1/alerts -> {alerts:[...]} + POST /api/v1/alerts <- {viewId, topic, label?} + DELETE /api/v1/alerts/{alert_id} + POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) + GET /api/v1/notifications -> {unread, items:[...]} + POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} + +The semantics — an alert is a view plus a remembered matched set, a notification is a NEW +ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. +This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. + +⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool +for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a +full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole +book, and the notification would name customers that user may not see — a permission leak wearing +a notification's clothes. The owner's own scope is the only correct basis for their alert. + +⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same +`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an +alert can see is by construction a row its owner could open. Re-implementing the filter here +would be a second definition of "matches", and those two would drift. +""" +import re + +from fastapi import APIRouter, Body, Depends + +import core.alerts as alerts +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. +_TOPICS = ("customer", "product") + +# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── +# +# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). +# +# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three +# keys onto the record at write time would give them to notifications minted AFTER the deploy and +# to nothing else — every notification already sitting in every tenant's inbox would open nothing, +# and the feature would be correct in the source and absent from the product +# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a +# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is +# another lane's file this wave — but that is the convenience, not the reason. +# +# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them +# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets +# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding +# here means the client branches on ONE field instead of re-deriving the same split. +# +# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` +# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by +# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) +# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, +# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review +# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real +# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored +# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary +# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed +# at was deleted two waves ago. + +#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them +#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). +NOTIF_KIND_ALERT = "alert" +NOTIF_KIND_AUTOMATION = "automation" +NOTIF_KIND_SHARE = "share" + +#: C3's `target.module` vocabulary, and the automation sub-selection. +TARGET_MODULE_DATABASE = "database" +TARGET_MODULE_AUTOMATION = "automation" +TARGET_TAB_RUNS = "runs" + +#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with +#: no producer is a string that reads as a feature — the reason this constant is named here and +#: cited from `routes_shares` rather than typed twice). +SHARE_TOPIC = "share" + +#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. +AUTOMATION_TOPIC = "automation" + +_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") + + +def route_for_topic(topic): + """A grid SCOPE key -> the registry route that renders it, or None. + + ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED + (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only + pair that differ — the registry names the surface (`customer_data`) while the grid names the + scope (`customer`) — so a topic passed through as a route sends every click to a page that + does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT + rather than plausible, because an absent target renders as a row that does not pretend to be + clickable, and a wrong one renders as a click that silently goes nowhere. + """ + t = str(topic or "").strip() + if t == "customer": + return "customer_data" + if t == "product": + return "product_data" + if _UT_TOPIC.match(t): + return t + return None + + +def _refusal_code(exc): + """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. + + ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` + (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own + cause. Anything that reduces all four to one word is throwing away the only information the + reader could have acted on. Returns `""` for a plain exception, so a caller can tell + "refused, and here is why" apart from "broke, and we do not know why". + """ + detail = getattr(exc, "detail", None) + if isinstance(detail, dict): + inner = detail.get("error") + if isinstance(inner, dict): + return str(inner.get("code") or "") + return "" + + +def notification_view(item): + """One STORED notification -> the shape the Inbox renders. PURE, and total. + + Never raises and never drops a row: an item it cannot classify comes back as an `alert` with + no `target`, which the client renders as an unclickable row rather than hiding. An inbox that + silently omits what it does not understand is the one failure a reader cannot detect. + """ + if not isinstance(item, dict): + return item + topic = str(item.get("topic") or "").strip() + alert_id = str(item.get("alertId") or "").strip() + + # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic + # says `automation` but whose producer key never arrived (a truncated payload, a server + # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work + # and silently does not, which is this repo's most-repeated failure shape. Failing the test + # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. + if topic == AUTOMATION_TOPIC and alert_id: + kind = NOTIF_KIND_AUTOMATION + target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} + elif topic == SHARE_TOPIC and alert_id: + # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, + # `row_id=`. + # + # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it + # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the + # target read `{module: "database", id: "view_42"}` — an instruction to open a database + # called `view_42`. It looked right in the payload and would have opened nothing. The + # producer resolves the object to its topic and hands over the route; this branch only + # shapes what it is given. + kind = NOTIF_KIND_SHARE + row_id = str(item.get("rowId") or "").strip() + target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, + **({"tab": row_id} if row_id else {})} + else: + kind = NOTIF_KIND_ALERT + route = route_for_topic(topic) + view_id = str(item.get("viewId") or "").strip() + target = None if route is None else ( + {"module": TARGET_MODULE_DATABASE, "id": route, + **({"tab": view_id} if view_id else {})}) + + # The email split: `subject` is the HEADER (what this is about — the alert, the automation, + # the database), `label` stays the BODY (what happened — the record that entered, the run + # summary). They were one field, which is why a notification read as a sentence with no + # sender and the pane could not be laid out like mail. + subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() + # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it + # is D-101's dead one; leaving it through would give the client two vocabularies for one + # question, which is the defect this wave's item 6 is about in a different file. + # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. + # + # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was + # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — + # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in + # the model or in the markup. Mail has a from. This is it. + # + # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. + # An alert firing and an automation landing rows have no person behind them; their honest + # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a + # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and + # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with + # you") — a sender recovered by regexing a sentence breaks the first time the sentence is + # reworded, and it would break silently, in the header. + # + # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an + # empty from column reads as a broken inbox rather than as an old notification. + actor = str(item.get("actor") or "").strip() + if kind == NOTIF_KIND_SHARE: + sender = actor or "A teammate" + elif kind == NOTIF_KIND_AUTOMATION: + # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's + # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` + # runs FIRST, so this server literal won and every actor-less automation notification + # showed the retired module name in the inbox's From column. + sender = actor or "Agents" + else: + sender = actor or "Alerts" + out = {**item, "read": bool(item.get("read")), "kind": kind, + "subject": subject or "Notification", "sender": sender} + if target is not None: + out["target"] = target + return out + + +def inbox_view(box): + """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. + + ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a + recount here would make the badge a function of whatever this page happened to include, which + is the exact defect `alertsModel.parseInbox`'s own header records from the other side. + """ + if not isinstance(box, dict): + return box + items = box.get("items") + if not isinstance(items, list): + return box + # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client + # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. + # + # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and + # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH + # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" + # against a browser clock would re-introduce the drift the offset exists to remove — a reader a + # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands + # now come from the same machine. + # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — + # the note two lines up records what happened last time only one of them was enriched. + return {**box, "now": alerts._now_iso(), + "items": [notification_view(n) for n in items]} + + +def _view_by_id(g, view_id): + """One saved view out of an assembly, by id. `None` when there is no such view. + + ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS + FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND + `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes + it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on + it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the + `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** + + ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In + `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare + FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our + side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above + the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical + line is swallowed by `/notifications`' `except Exception: continue`, so **every stored + view-alert was silently dropped from the Inbox** and nobody had anything to report at all. + One expression, one loud symptom and one silent one. + + ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site + be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. + + ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was + keyed `{id: view}` — which is precisely why the gate was green while production raised on + every call. The fixture is moving to the production shape in this same change, and tolerating + both here means a caller that legitimately holds one cannot resurrect the bug. + """ + want = str(view_id or "") + if not want: + return None + views = (g or {}).get("views") + if isinstance(views, dict): + found = views.get(want) + return found if isinstance(found, dict) else None + if not isinstance(views, list): + return None + for v in views: + if isinstance(v, dict) and str(v.get("id") or "") == want: + return v + return None + + +def _topic_or_400(raw): + topic = str(raw or "").strip().lower() + if topic.startswith("ut_") or topic in _TOPICS: + return topic + raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") + + +def _owner_session(session: Session, owner: str): + """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). + + ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an + owner session is built by swapping the `user` RECORD and letting both derive themselves. An + earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the + first write hook of the wave; the properties are the single definition of who a session is, + and going around them is how a session with an admin flag and a non-admin record exists. + + Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather + than evaluating as somebody else, which is the fail-closed direction. + """ + import core.users as users + + if str(owner) == str(session.uname): + return session + rec = (users.registry() or {}).get(str(owner)) + if not isinstance(rec, dict) or not rec.get("active", True): + return None + # `_public` is THE definition of what a session may know about its own account (never a hash + # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a + # second definition, and the one that leaks is always the copy. + return Session(tenant=session.tenant, user=users._public(str(owner), rec), + claims=session.claims, runtime=session.runtime) + + +def _evaluate(session: Session, rec: dict, assemblies=None): + """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. + + ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole + of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each + one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two + alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, + because each `_evaluate` was a closed call. + ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module + note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak + wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and + must never share an entry. Getting that key wrong is the one way this optimisation could leak. + ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: + they evaluate ONE alert and a memo for a single call is pure overhead. + """ + import aios_grid + from harness import filter_eval + + owner_sess = _owner_session(session, rec.get("owner")) + if owner_sess is None: + return {"skipped": "owner_unavailable"} + topic = str(rec.get("topic") or "") + memo_key = (topic, str(owner_sess.uname)) + g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None + if g is None: + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. + # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the + # one-shot field-name correction acks for every `ut_` topic that has an alert — + # taking them from the `/workspace` refresh that exists to show them to the person + # who made the edit. The customer branch below has always passed False; this one + # inherited a default nobody re-read. An inbox poll must never consume a one-shot. + g = ut_assembly(owner_sess, topic, + storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", + consume_corrections=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) + except Exception as e: # noqa: BLE001 + # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not + # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it + # keeps a blanket catch — but it now reports the refusal's OWN code where there is + # one. `type(e).__name__` said `HTTPException` for four different causes, and + # `lastError` is the only place a user ever learns why an alert stopped firing. + # + # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an + # evaluation genuinely needs the rows to run the filter over. So an alert on a + # read-through grid is created (T22) and then skips at evaluation with + # `window_required` naming why — which is D-184's remaining half, and it is a + # SENTENCE now rather than silence. + return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} + if isinstance(assemblies, dict): + assemblies[memo_key] = g + + view = _view_by_id(g, rec.get("viewId")) + if not isinstance(view, dict): + # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than + # deleting the alert: an alert that silently vanishes is indistinguishable from one that + # never fires, and the user cannot debug what is not there. + return {"skipped": "view_missing"} + + # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived + # and overlay values on a row. Evaluating a filter against raw pool dicts would silently + # never match any condition on a user-created or measure column. + # + # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND + # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. + # + # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows + # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner + # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: + # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell + # evaluated as BLANK. Measured on one assembly, same view, same rows: + # rows_src state='unpaid' / 'paid' + # _evaluate saw state='' / '' ⇐ every base cell blank + # the GRID saw state='unpaid' / 'paid' + # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` + # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it + # inverted.** End to end: a row whose value arrived by import, automation, paste or the create + # door never fired; only a value typed as a hand EDIT did. + # + # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; + # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); + # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns + # `skipped: window_required`). + # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it + # would let a stale definition value shadow an edit the user has just made — the same defect + # `table_rows`' own note records, arriving from the other side. + _ov = (g.get("ws") or {}).get("overlays") or {} + _merged = {} + for _r in g["rows_src"]: + _pid = str(_r.get("pid")) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _o = _ov.get(_pid) + if isinstance(_o, dict): + _cells.update(_o) + _merged[_pid] = _cells + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, + derived=g.get("derived")) + config = view.get("config") or view + ctx = filter_eval.EvalCtx( + cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} + for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, + measure_sets=g.get("measure_sets") or {}, + today=g.get("today")) + pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, + member_pids=config.get("memberPids")) + labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} + return alerts.evaluate(rec.get("id"), [str(p) for p in pids], + labels=labels, partial=False, st=session.runtime) + + +@router.get("/alerts") +def list_alerts(session: Session = Depends(require_session)): + return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime)} + + +@router.post("/alerts") +def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): + body = body or {} + view_id = str(body.get("viewId") or "").strip() + if not view_id: + raise err(400, "bad_view", "an alert needs the id of the view it watches") + topic = _topic_or_400(body.get("topic")) + _require_filtered_view(session, topic, view_id) + import uuid + aid = f"al_{uuid.uuid4().hex[:12]}" + rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, + label=body.get("label") or "", st=session.runtime) + # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". + # Deferring this to the first write hook would mean the next edit announces the whole view. + outcome = _evaluate(session, rec) + return {"alert": {**rec, "seeded": True}, "first": outcome} + + +def _require_filtered_view(session: Session, topic: str, view_id: str): + """400 unless `view_id` exists on `topic` AND actually narrows something. + + ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one + that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" + (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an + entrant again — there is nothing left to enter. The owner's words are *"when a Record gets + into that Filter's criteria"*: no criteria, no alert, and said at creation rather than + discovered by never being notified. + + `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a + half-typed rule is not a filter, and this must agree with what actually narrows or it would + accept a view whose one rule the engine then ignores. + + ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and + the second one hid the first. + + (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is + `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a + read-through grid built the whole pool — and `scoped_pool` refuses that with + `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` + (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, + runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, + closed** — an alert on a read-through grid can now be made at all. + (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` + is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and + `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by + *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED + A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design + (a 5xx body is the server's internals), substituting *"Something went wrong on our + side."* — which is the owner's screenshot, word for word. A knowable cause returned as a + 5xx is invisible by construction, so re-wording the 503 could never have fixed this. + ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is + honest. What it may no longer do is catch a refusal that already knows its own name. + """ + from fastapi import HTTPException + + from harness import filter_eval + + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⚠ `consume_corrections=False` — the customer branch has always passed it and this + # one inherited a default nobody re-read. Creating an alert must not eat the one-shot + # field-name correction acks belonging to the `/workspace` refresh that exists to show + # them to the person who made the edit. Same defect `_evaluate`'s header records. + g = ut_assembly(session, topic, + storage_key=f"{session.tenant}:{topic}:{session.uname}", + consume_corrections=False, with_rows=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(session, scope=topic, consume_corrections=False) + except HTTPException: + raise # it already names its own cause + except Exception as e: # noqa: BLE001 + # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, + # the one path that reaches this branch is also the one path with nothing to debug from. + raise err(503, "unavailable", + f"the table could not be read ({type(e).__name__}) — try again in a moment") + view = _view_by_id(g, view_id) + if not isinstance(view, dict): + raise err(404, "no_view", "that view does not exist on this table") + nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) + + # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT + # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS + # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have + # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines + # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the + # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. + # + # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the + # fix was right, and the three reasons it hid are the most transferable thing in this file. + # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each + # naming themselves as the origin of the same screenshot are mutually exclusive, and the next + # reader believes whichever they meet first — which is why this correction sits above rather + # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead + # of the working tree. [[grep-output-is-not-source]] + # + # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, + # that this was owner item 17 — see the correction directly above). + # + # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other + # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF + # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. + # + # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the + # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. + # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever + # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that + # worked was the refusal path: "Alert me about new records" had never once created an alert + # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the + # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went + # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. + # + # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check + # arity until the line RUNS, and this line runs only on the success path of a feature whose + # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, + # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal + # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, + # thorough and honest about everything except the one path the feature exists for. + # + # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a + # column up in; building a second dict here would be a second answer to one question, which + # is this wave's other headline defect in a different file. Its leading underscore is a real + # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. + columns = filter_eval._columns_map(g.get("fields") or []) + + def _any_active(ns): + for n in ns or (): + if isinstance(n, dict) and isinstance(n.get("children"), list): + if _any_active(n["children"]): + return True + elif filter_eval.is_rule_active(n, columns): + return True + return False + + if not _any_active(nodes): + raise err(400, "no_filter", + "this view has no active filter, so no record can ever ENTER it — add a " + "condition to the view first, then create the alert") + + +@router.delete("/alerts/{alert_id}") +def delete_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + if str(rec.get("owner")) != str(session.uname) and not session.admin: + raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") + alerts.delete(alert_id, st=session.runtime) + return {"ok": True} + + +@router.post("/alerts/{alert_id}/run") +def run_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + return _evaluate(session, rec) + + +@router.get("/notifications") +def notifications(session: Session = Depends(require_session)): + """The inbox — RE-EVALUATED on read, which is a deliberate design choice. + + ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The + plan was a push hook: the automation engine calls `after_write` when it lands rows. But + `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be + evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside + a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity + construction that leaks scope. + + Pulling on read has none of that: the caller IS a session, the assemblies are already + scope-cached, and the user cannot observe the difference — an inbox is only ever read by + someone opening it. The cost is that a notification is minted when you LOOK rather than when + the row landed, so the `at` stamp is detection time, not arrival time. + + `after_write` stays exported for the day the engine can hand over a real identity. + + ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. + ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is + **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** + (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the + re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the + body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole + grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which + is the difference between "fine" and "three seconds per alert" the day somebody uses the + feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what + this does: every alert is still evaluated, against the same rows, in the same order. + """ + assemblies = {} + for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): + try: + _evaluate(session, rec, assemblies=assemblies) + except Exception: # noqa: BLE001 + continue # one bad alert must not empty the pane + # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before + # this wave carries a `target` too. See `notification_view`'s header for why it is derived. + return inbox_view(alerts.inbox(session.uname, st=session.runtime)) + + +@router.post("/notifications/read") +def read_notifications(body: dict = Body(default=None), + session: Session = Depends(require_session)): + body = body or {} + ids = body.get("ids") + if ids is not None and not isinstance(ids, list): + raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") + # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox + # module re-renders from it — an un-enriched answer here would strip `target` off every row + # the moment somebody marked one read, i.e. the feature would work until first use. + return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), + st=session.runtime)) + + +def after_write(session: Session, topic_key: str): + """THE WRITE HOOK — call after a write that could change what a view matches. + + Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation + upserts reach it the same way. It never raises: an alert evaluation failing must not fail the + edit that triggered it. + + ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes + one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. + ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the + push hook was resolved the other way). Booked rather than wired: minting a session inside the + engine's worker thread is the ad-hoc identity construction this file exists to avoid. + """ + try: + assemblies = {} + return alerts.after_write(topic_key, st=session.runtime, + runner=lambda rec: _evaluate(session, rec, + assemblies=assemblies)) + except Exception: # noqa: BLE001 + return {"evaluated": 0} diff --git a/api/routes_automation.py b/api/routes_automation.py index 0b8aad0a997592d356b825513ee8877b046e066b..88849c492b7823d3b84c96c4db87725749c92132 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -14,6 +14,7 @@ trigger becomes a public one — the same class of mistake as an empty-200 permi import os import re import time +from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends, Header, Request @@ -53,7 +54,7 @@ MODULE = "automation" _GATE = module_gate(MODULE) -def _wire(defn, tenant): +def _wire(defn, tenant, rt=None): """One automation, as the client reads it. `running` is PROCESS state, never store state — see the engine header on why a persisted 'running' is a permanent lock.""" live = engine.running(tenant, defn.get("id")) @@ -141,6 +142,16 @@ def _wire(defn, tenant): # back. What needs the target database's schema (an enrich binding resolved by the profile # FLAG) stays a run-time refusal — see `ACTION_REQUIRED`'s note. "unconfigured": engine.unconfigured_actions(defn), + # ⭐⭐ WAVE 36 · W36-T38 (owner item 8 / R9) — WHICH ACTIONS AN AGENT BUILT. + # `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation. + # ⛔ THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to + # draw a lock and explain who owns the step, never to withhold what the step does — an + # automation nobody can audit is worse than one nobody can edit. + # ⚠ `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the + # runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of + # an absent prop the client silently reads as undefined + # [[flag-shipped-without-its-writer]]. + "agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {}, } @@ -243,7 +254,7 @@ def _field_agent_rows(session): # — the exact way `awaitingResults` shipped inert for a whole wave. Passing the # synthetic DEFINITION through the same function makes them identical by # construction; only `system` is stamped afterwards, because no stored row has it. - row = _wire(defn_syn, session.tenant) + row = _wire(defn_syn, session.tenant, rt=session.runtime) row["system"] = SYSTEM_FIELD_AGENT out.append(row) return out @@ -334,7 +345,7 @@ def _odoo_sync_row(session, detail=False): "config": {"connector": "odoo", "every": every, "frozen": frozen}}]}, } - row = _wire(defn, session.tenant) + row = _wire(defn, session.tenant, rt=session.runtime) row["system"] = SYSTEM_ODOO_SYNC # ⛔ THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence` # speaks the automation vocabulary — it builds "When , run N actions" out of @@ -745,7 +756,7 @@ def list_automations(session: Session = Depends(_GATE)): pass _STATEMENTS_SEEDED.add(session.tenant) defs = engine.all_definitions(session.runtime) - items = [_wire(d, session.tenant) for _, d in + items = [_wire(d, session.tenant, rt=session.runtime) for _, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] # ⭐⭐ WAVE 34 · CONTRACT C3 (W34-T48) — field agents join the list as SYNTHETIC rows. # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be @@ -1133,6 +1144,17 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends prompt = str((body or {}).get("prompt") or "").strip() if not prompt: raise err(400, "no_prompt", "type what you want the automation to do") + # ⭐⭐ ASK D-18 (2026-08-18) — THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off + # the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that + # configured nothing: the value was accepted and dropped, which is the shape three tickets in + # this wave already tripped over. D held T34 rather than mark a picker BUILT while its value + # stopped at the browser, and was right to. + # ⚠ UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture, + # and C5's everywhere else): a model the ladder stops offering must not turn every later draft + # into an error. The response already names the rung that ANSWERED, which is the honest half. + _model = str((body or {}).get("model") or "").strip().lower() + if _model in ("", "auto"): + _model = None # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open` # per table, so the model is shown exactly the databases this caller may already see and cannot # name one they were not granted — the permission wall re-used, never a second one built beside @@ -1170,6 +1192,7 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends # sites, this one has a real person behind it: somebody typed the sentence, so `user` is # the caller rather than the automation's owner. st=session.runtime, user=getattr(session, "uname", "") or "", + model=_model, chat=_DRAFT_CHAT[0]) if refusal or not draft: raise err(400, "draft_refused", refusal or "no automation could be drafted from that") @@ -1263,6 +1286,244 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends "provider": provider, "dropped": dropped, "notes": _notes, "saved": False} +# ══════════════════ WAVE 36 · W36-T38 (owner item 8, ruling R9) — AGENT-AUTHORED ACTIONS ══════ +# +# Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any +# 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a +# script etc. We need to really guardrail the reach of this script."* +# +# ⭐⭐ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the +# user MAY read an agent-authored action's configuration and MAY delete it; what they may not do +# is hand-edit it. Hiding a config from the person whose workspace it runs in is a different +# product, and a worse one — nobody can audit what they cannot see. +# +# ⛔ AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already +# carries that lesson for a different control; a client-side lock is a suggestion, and the whole +# point of R9 is that the agent's configuration stays coherent with what the agent believes it +# built. +# +# ⚠ WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every +# action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) — so a +# marker stored inside the action would be silently erased on the next save, and the wall would +# quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by +# `(automation id, action id)`; the configuration itself stays where it always was, in the +# automation, with exactly one writer. +AGENT_ACTIONS_KEY = "automation_agent_actions" + +#: Ids this door mints. ⚠ It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine +#: re-mints it and the annotation points at an action that no longer carries that id. +_AGENT_ACTION_PREFIX = "act_ag" + + +def _agent_marks(rt): + """`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure — an unreadable + annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail. + + ⛔ AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because + it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a + lost mark lets a person edit a config they own anyway, in their own workspace, on a step they + could always have deleted outright. A wall that failed CLOSED would instead make an automation + permanently unsavable because one annotation read timed out. + """ + try: + found = rt.get(AGENT_ACTIONS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _flat_actions(actions, out=None, depth=0): + """Every action in a flow, INCLUDING the ones nested inside a group's arms. + + ⚠ NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the + same reason: a step inside an If/then branch is exactly the one a person cannot see, and a + wall that only looked at the top level would leave the agent's own nested step editable. + """ + out = [] if out is None else out + if depth > 6 or not isinstance(actions, list): + return out + for action in actions: + if not isinstance(action, dict): + continue + out.append(action) + for key in ("then", "else", "actions"): + _flat_actions(action.get(key), out, depth + 1) + return out + + +def _flow_actions(defn): + return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or []) + + +def _same_action(left, right, rt): + """Do these two actions carry the SAME kind and configuration? + + ⛔ COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and + a nuisance. The stored action has already been through the cleaner; a client round-trip has + not, so it may carry a key order, a blank string or a dropped condition that means nothing. + Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a + no-op is one an operator learns to route around [[one-question-two-normalizers]]. + """ + def _clean(action): + cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt) + if error or not cleaned: + return None + one = dict(cleaned[0]) + one.pop("id", None) + return one + + a, b = _clean(left), _clean(right) + return a is not None and a == b + + +def _agent_owned_guard(session, auto_id, body): + """R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration. + + Three outcomes, and the middle one is the ruling: + * the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it + * the action is present and CHANGED -> **409**, naming the action and the agent + * the action is present and identical -> nothing happens, so an ordinary save of a + flow that merely CONTAINS an agent action + is not refused + """ + marks = _agent_marks(session.runtime).get(str(auto_id)) or {} + if not isinstance(marks, dict) or not marks: + return + incoming = (body or {}).get("flow") + if not isinstance(incoming, dict) or "actions" not in incoming: + return + stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + was = {str(a.get("id") or ""): a for a in _flow_actions(stored)} + now = {str(a.get("id") or ""): a + for a in _flat_actions(incoming.get("actions") or [])} + for action_id, mark in marks.items(): + action_id = str(action_id) + if action_id not in now or action_id not in was: + continue # removed, or never stored: not an EDIT + if _same_action(was[action_id], now[action_id], session.runtime): + continue + who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent") + raise err(409, "agent_owned", + f"this step was built by {who} and only {who} can change how it is set up. " + f"You can read it, and you can delete it, but it cannot be edited by hand") + + +def _prune_marks(session, auto_id): + """Drop annotations whose action is gone, and the whole entry when the automation is. + + ⚠ CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever + grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a + deleted automation would sit in the tenant document forever. + """ + auto_id = str(auto_id) + + def _set(cur): + cur = dict(cur or {}) + marks = cur.get(auto_id) + if not isinstance(marks, dict): + cur.pop(auto_id, None) + return cur + defn = (engine.all_definitions(session.runtime) or {}).get(auto_id) + if defn is None: + cur.pop(auto_id, None) + return cur + alive = {str(a.get("id") or "") for a in _flow_actions(defn)} + kept = {k: v for k, v in marks.items() if str(k) in alive} + if kept: + cur[auto_id] = kept + else: + cur.pop(auto_id, None) + return cur + + session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") + + +@router.post("/automations/{auto_id}/agent-actions") +def agent_author_action(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + """THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9). + + {agent: "", action: {kind, config, when?, id?}} + -> {automation, actionId, agent} + + ⛔ THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail + the reach of this script" means in code: an agent cannot invent a config key, cannot name a + kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The + agent gets a different WALL on editing, never a wider vocabulary. + + ⛔ AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the + catalog, so `clean_actions` refuses it here exactly as it refuses it for a person — see this + lane's mailbox for why the script ARM is booked rather than half-built: a per-row script + contract and a client card are both missing, and a step that reports success and does nothing + is the failure this repo has already paid for. + + ⚠ THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does + not exist would refuse every future edit with a sentence naming nobody. + """ + import routes_slack # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + agent_id = str(body.get("agent") or "").strip() + agent = routes_slack._agents(session.runtime).get(agent_id) + if not isinstance(agent, dict): + raise err(404, "no_agent", "there is no agent with that id in this workspace") + action = body.get("action") + if not isinstance(action, dict) or not str(action.get("kind") or "").strip(): + raise err(400, "no_action", "an action needs a kind") + + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + if str(defn.get("system") or "").strip(): + raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"]))) + + existing = list(((defn.get("flow") or {}).get("actions") or [])) + taken = {str(a.get("id") or "") for a in _flat_actions(existing)} + action_id = str(action.get("id") or "").strip() + if not action_id or action_id not in taken: + n = 1 + while f"{_AGENT_ACTION_PREFIX}{n}" in taken: + n += 1 + action_id = f"{_AGENT_ACTION_PREFIX}{n}" + fresh = dict(action, id=action_id) + + # ⚠ VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was. + checked, error = engine.clean_actions([fresh], rt=session.runtime) + if error or not checked: + raise err(400, "invalid_action", error or "that action could not be built") + + replaced = False + for i, one in enumerate(existing): + if isinstance(one, dict) and str(one.get("id") or "") == action_id: + existing[i], replaced = fresh, True + break + if not replaced: + existing.append(fresh) + + notes = [] + updated, error = engine.patch(session.runtime, str(auto_id), + {"flow": {**(defn.get("flow") or {}), "actions": existing}}, + username=session.uname, notes=notes) + if error: + raise err(400, "invalid_automation", error) + + def _set(cur): + cur = dict(cur or {}) + marks = dict(cur.get(str(auto_id)) or {}) + marks[action_id] = {"agent": agent_id, + "agentName": str(agent.get("label") or agent.get("channelName") + or agent_id), + "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "by": session.uname} + cur[str(auto_id)] = marks + return cur + + session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") + _prune_marks(session, auto_id) + return {"automation": _wire(updated, session.tenant, rt=session.runtime), + "actionId": action_id, "agent": agent_id, "notes": notes} + + @router.get("/automations/{auto_id}") def get_automation(auto_id: str, session: Session = Depends(_GATE)): """One automation, by id. @@ -1295,7 +1556,7 @@ def get_automation(auto_id: str, session: Session = Depends(_GATE)): defn = engine.all_definitions(session.runtime).get(str(auto_id)) if defn is None: raise err(404, "unknown_automation", "no automation with that id") - return {"automation": _wire(defn, session.tenant)} + return {"automation": _wire(defn, session.tenant, rt=session.runtime)} @router.post("/automations") @@ -1317,7 +1578,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend notes=_notes) if error: raise err(400, "invalid_automation", error) - return {"automation": _wire(defn, session.tenant), "notes": _notes} + return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} @router.patch("/automations/{auto_id}") @@ -1339,6 +1600,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None), return _patch_odoo_sync(session, body or {}) # D-75, same channel as `create` above — an EDIT is where this matters most, because the # person has just typed the thing that gets dropped. + # ⛔⛔ W36-T38 / R9 — THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled` + # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the + # step and permits an ordinary save of a flow that merely contains one. + _agent_owned_guard(session, auto_id, body or {}) _notes = [] defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname, notes=_notes) @@ -1346,7 +1611,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None), raise err(400 if error != "no such automation" else 404, "invalid_automation" if error != "no such automation" else "unknown_automation", error) - return {"automation": _wire(defn, session.tenant), "notes": _notes} + # An action the user removed takes its annotation with it, or the mark outlives its subject + # and refuses an id nobody can see. + _prune_marks(session, auto_id) + return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} @router.delete("/automations/{auto_id}") @@ -1384,6 +1652,9 @@ def delete_automation(auto_id: str, session: Session = Depends(_GATE)): if _sys: raise err(409, "system_agent", engine.system_agent_refusal(_sys)) engine.remove(session.runtime, auto_id) + # R9's other half: the user MAY delete an agent-authored step, and deleting the whole + # automation takes its annotations with it rather than stranding them in the document. + _prune_marks(session, auto_id) return {"deleted": str(auto_id)} @@ -1439,7 +1710,7 @@ def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depend raise err(404 if error == "no such automation" else 400, "unknown_automation" if error == "no such automation" else "node_not_toggleable", error) - return {"automation": _wire(defn, session.tenant)} + return {"automation": _wire(defn, session.tenant, rt=session.runtime)} @router.post("/automations/{auto_id}/hook/{token}") diff --git a/api/routes_grid.py b/api/routes_grid.py index 6c7609921247127cfa6dc7a7880db7a3fd45bbcd..27cec4519f43ae113fb2a06b685bff9fbeaceb7c 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -1000,3 +1000,84 @@ def grid_events_route(body: dict = Body(default=None), # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh # data that was never in it. return out + + +# ── CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS ─────────────────────────────────── +# ⭐⭐ R6 — *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table` +# is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never +# imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid` +# behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as +# `routes_odoo_tables` declares connected tables to `user_tables.register_connected`. +# +# ⛔ REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files +# are outside wave 36's lane-C fence. The readers themselves are three lines each and call the +# SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no +# second scope derivation — only a second CALLER of the one that exists. +# +# ⚠ AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` + +# `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and +# `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done +# here, because neither file is in this fence. What ships now is the arm the wave is load-bearing +# on — every `ut_*` database, plus E's sandbox — and a topic arm that is REAL rather than stubbed, +# so `scoped_table`'s topic leg is exercised by the product instead of only by a gate. +def _topic_rt(st, module): + """The tenant runtime a topic pool must be built against, or a REPORTED refusal. + + ⛔ A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the + default" without picking a tenant at random — which on this box is tenant #0's PRODUCTION + data. Standing rule 1's second sentence: say why, and say what to do instead. + """ + if st is None: + import core.perm_scope as perm_scope + raise perm_scope.Unresolvable( + subject="rows", effect="unreadable", + cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant " + f"runtime was passed", + recommendation="pass the session's runtime as `st=`. A topic pool cannot be " + "resolved without knowing which tenant is asking") + return st + + +def _customer_rows(table_key, user, st): + """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" + import aios_grid + import core.perm_scope as perm_scope + from routes_customers import _pool_for + + rt = _topic_rt(st, table_key) + team_id, agent = perm_scope.derive_pool_scope(user, table_key) + return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) + + +def _product_rows(table_key, user, st): + """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a + BU-pinned reader gets that BU's field contract rather than the consolidated one.""" + import core.perm_scope as perm_scope + from routes_products import _pool_for, pd_fields + + rt = _topic_rt(st, table_key) + team_id, _agent = perm_scope.derive_pool_scope(user, table_key) + return pd_fields(consolidated=team_id is None), _pool_for(rt, team_id) + + +def _register_topic_rows(): + """Declare both topic readers to C1. Called at import; returns the registered key set. + + ⚠ THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at + module import, and `from routes_products import MODULE` here would pull a sibling router in + before its own imports have settled. Every other cross-router reference in this file is lazy + for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot + drift into naming a topic that does not exist. + """ + import core.perm_scope as perm_scope + + perm_scope.register_rows(_customer_rows, MODULE) + return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE) + + +#: ⚠ `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE` +#: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same +#: thing and be a different one — [[constant-two-features-share]] waiting to happen. +_PRODUCT_MODULE = "product_data" + +_C1_ROW_SOURCES = _register_topic_rows() diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 0d8534c02e4acf8e5318379694ff5a39602887f5..4ef61ca06030bf7abf2238ae1421cbde6686560d 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -1,1073 +1,1073 @@ -"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). - -Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER -return a decrypted field — list rows carry a masked preview, and the decrypt function is a -connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's -env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** -count (rows holding overlay data whose pids the current pool no longer serves; counted and -drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly -in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with -the keychain cutover wave R3 staged. -""" -import os - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session -# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door -# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall -# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per -# entry instead of per endpoint. `session.admin` is still what business-wide requires. - -router = APIRouter(prefix="/api/v1") - -#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by -#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another -#: freezes nothing while reporting success, which is the shape of the bug D-10 books. -from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 - ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) -#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED -#: odoo connector is paused; read by the pool path while paused; survives a Space restart. -SNAPSHOT_KEY = "connector_snapshots" - -#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. -#: -#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves -#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and -#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's -#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's -#: connection, and handing it to another tenant is the leak this method exists to prevent."* -#: -#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and -#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value -#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, -#: the gate reds instead of the product quietly disagreeing with itself. -ENV_ODOO_TENANT = "royal-imports" - - -def env_odoo_available(rt): - """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) - - ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. - `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant - guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was - `connected` and offered "Manage keys" for a credential belonging to another company. This - module asked the same question correctly two functions below, which is the whole shape of - [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in - as the second tenant. - - ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which - source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, - wrong for "should the environment row be listed at all", which is what the connectors pane - needs in order to show an inactive env source beside an active keychain one. - - Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is - how another tenant's environment got reported as this tenant's connection in the first place. - """ - try: - if not (os.environ.get("ODOO_URL") or "").strip(): - return False - return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT - except Exception: # noqa: BLE001 - return False - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. -# Every keychain entry and connector carries a scope, selectable per connection; business-wide is -# admin-only and applies to every user in the tenant."* -# -# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file -# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is -# an access rule, not a secret, so it lives in the layer that already decides who may ask. -# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves -# cannot drift into two vocabularies. -SCOPES = ("business", "personal") -#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before -#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. -#: Materialising it would be a migration nobody authorised and would rewrite the store on the next -#: read [[a-migration-that-runs-on-the-next-write]]. -DEFAULT_SCOPE = "business" -#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. -#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean -#: the same thing and there is only one way to spell the common case. -SCOPE_KEY = "keychain_scopes" - -#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. -#: -#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* -#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is -#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they -#: would silently become the credential the entire workspace's databases are built from, which is -#: a credential elevation wearing a scope picker. -#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes -#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is -#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door -#: from the only side B owns, and the resolver-side guard is booked for A. -TENANT_WIDE_TYPES = ("odoo", "meta_ads") - - -def clean_scope(raw, default=DEFAULT_SCOPE): - """A scope word from the wire, or None when the caller said something we do not speak. - - Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole - reason this returns None rather than falling back: a typo'd `"personel"` silently becoming - business-wide is exactly the failure a scope picker exists to prevent. - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default - got = str(raw).strip().lower() - return got if got in SCOPES else None - - -def _scope_rows(rt): - try: - return dict(rt.get(SCOPE_KEY) or {}) - except Exception: # noqa: BLE001 - return {} - - -def entry_scope(rt, entry_id, rows=None): - """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a - census does not re-read the store once per entry.""" - r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) - if not isinstance(r, dict): - return DEFAULT_SCOPE, "" - return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) - - -def may_see(scope, owner, uname): - """R4's visibility rule: business-wide is everyone's, personal is its owner's. - - ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the - per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token - is identity, not infrastructure"*). An admin who could read every member's personal credential - would make "personal" a label rather than a boundary. - """ - return scope != "personal" or str(owner) == str(uname) - - -def visible_entries(rt, uname, is_admin=False): - """This USER's view of the keychain: every business entry plus their own personal ones, each - row carrying its `scope` and `owner` so no caller has to ask a second time. - - ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can - hold a connection of their own; it does not hand them four characters of the workspace's Odoo - key. So a business row a member did not create arrives WITHOUT `preview` — they can see that - the connection exists and is theirs to use, which is the whole of what R4 grants. The default - is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. - """ - rows = _scope_rows(rt) - out = [] - for e in _kc().list_entries(rt): - scope, owner = entry_scope(rt, e["id"], rows) - if not owner: - owner = str(e.get("createdBy") or "") - if not may_see(scope, owner, uname): - continue - row = {**e, "scope": scope, "owner": owner} - if not (is_admin or str(owner) == str(uname)): - row["preview"] = "" - out.append(row) - return out - - -def _write_scope(rt, entry_id, scope, owner): - """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the - default, so the store holds one spelling of the common case.""" - def _up(cur): - if scope == DEFAULT_SCOPE: - cur.pop(str(entry_id), None) - else: - cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} - return cur - - rt.update(SCOPE_KEY, _up, flush="sync") - return True - - -def _may_touch(session, row): - """May this session change or delete `row`? An admin owns the business-wide ones; a member - owns their own personal ones. Anything else is not theirs to move.""" - if row.get("scope") == "personal": - return str(row.get("owner") or "") == str(session.uname) - return bool(session.admin) - - -def _kc(): - import core.keychain as keychain - return keychain - - -def _resolved_odoo_key(rt): - """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the - shapes this module needs: the display string (`env` / `keychain:`) and the key the pause - flag is stored under. - - ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, - beside the `odoo_source()` it must agree with. This function had its own copy of the same - three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and - a second copy is exactly how a pause flag comes to be written against one resolution and read - against another, freezing nothing while the UI reports success. The two SHAPES stay here - because they are this module's presentation concern; the DECISION does not. - """ - flag_key = rt.odoo_flag_key() - if not flag_key: - return None, None - return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key - - -def odoo_paused(rt): - """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry - that is not the resolved source freezes nothing — it serves nothing. - - ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the - measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question - the customer pool does. This name stays because `routes_customers`, `routes_products` and - `verify_api` all call it — moving the logic without moving the door keeps one answer and - costs no caller a change. - """ - return bool(rt.odoo_paused()) - - -def _snap_scope_key(team_id, agent): - return f"t={team_id}|a={agent}" - - -def load_pool_snapshot(rt, team_id, agent): - """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider - scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" - try: - snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} - e = snap.get(_snap_scope_key(team_id, agent)) - if isinstance(e, dict) and isinstance(e.get("rows"), list): - return float(e.get("ts") or 0), e["rows"] - except Exception: - pass - return None - - -def save_pool_snapshots(rt, taken_by=""): - """Persist every currently-cached pool scope as the pause-time snapshot ('the last - successful sync', made concrete). Ensures the consolidated default scope exists first so - a pause on a cold process still captures something to serve.""" - import time as _time - import routes_customers as _rc - try: - _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on - except Exception: - pass # cold + Odoo down: persist whatever IS cached - pools = {} - for key, entry in list(rt.pool_cache.items()): - if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" - and isinstance(entry, tuple) and len(entry) == 2 - and isinstance(entry[1], list)): - pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} - if not pools: - return 0 - - def _up(cur): - cur["odoo_pool"] = pools - cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") - cur["takenBy"] = str(taken_by or "") - return cur - - rt.update(SNAPSHOT_KEY, _up, flush="sync") - return len(pools) - - -def _rel_reconnect(rt): - """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot - disagree about what "resume" means.""" - import odoo_relational as rel - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = False - cur.pop("frozenAt", None) - cur.pop("frozenBy", None) - return cur - - rt.update(rel.CONFIG_KEY, _up, flush="sync") - return True - - -# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION -# -# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as -# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so -# the risk is what this block is mostly about. -# -# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env -# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already -# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, -# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build -# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks -# empty while its Odoo grids work. -# -# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns -# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — -# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under -# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector -# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That -# is D-259's exact shape (a pause written under one key and read under another) and -# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the -# SAME pass, and the carry is asserted. - - -def _env_odoo_fields(): - """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. - - ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. - Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a - PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo - offline, on a deployment where it had been working. The env fallback cannot save it, because the - entry's existence is what turns the fallback off. - ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the - field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two - vocabularies meet here; nowhere else. - """ - want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), - ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) - got = {field: (os.environ.get(env) or "").strip() for field, env in want} - missing = sorted(env for field, env in want if not got[field]) - if missing: - return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " - f"would take this tenant's Odoo offline rather than migrate it") - return got, "" - - -def migrate_env_odoo(rt): - """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. - - `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every - path including the skips, because "already migrated", "no keychain key on this deployment" and - "the env is incomplete" are three different operator actions. - - ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, - measured three times: a developer's CLI write to the tenant store is reverted by the running - Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success - every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here - would be a dry run that lies, and the thing it would lie about is a credential. - - ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a - third-party failure must not take the boot down. - """ - out = {"done": False, "entry": "", "carried_pause": False, "why": ""} - if not env_odoo_available(rt): - # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. - out["why"] = "this tenant has no environment Odoo credential to migrate" - return out - fields, why = _env_odoo_fields() - if not fields: - out["why"] = why - return out - # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the - # NEW key and the old one is unreachable through the resolver — so the only moment this fact can - # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. - try: - was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) - .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) - except Exception: # noqa: BLE001 - was_paused = False - row, why = _kc().ensure_entry_of_type( - rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") - if row is None: - out["why"] = why - return out - out["done"], out["entry"] = True, row["id"] - if was_paused: - # The freeze followed the credential. Without this the connector silently RESUMES pulling - # live Odoo at the first boot after the migration. - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - entry = dict(cur.get(row["id"]) or {}) - entry["paused"] = True - entry["pausedBy"] = "system" - entry["pausedNote"] = ("carried over from the environment source when the credential " - "was migrated onto the keychain") - cur[row["id"]] = entry - return cur - - try: - rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") - out["carried_pause"] = True - except Exception as exc: # noqa: BLE001 - # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse - # than one that did not run, so this is the one failure that must never be silent. - out["why"] = (f"the credential migrated but the PAUSE could not be carried over " - f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") - return out - - -def _own_row(session, entry_id): - """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the - caller cannot see: telling a member that somebody else's personal credential EXISTS is the - disclosure the scope is for.""" - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(entry_id)), None) - if row is None: - raise err(404, "no_entry", "no such key") - return row - - -@router.get("/admin/keychain") -def list_keychain(session: Session = Depends(require_session)): - """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. - R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would - ship the feature and no door to it. The wall moved INTO the payload: a member sees the - business-wide entries and their own, never anybody else's personal one.""" - kc = _kc() - return {"entries": visible_entries(session.runtime, session.uname, - bool(session.admin)), - "locked": not kc.unlocked(), - #: the vocabulary and the permission, so the client renders a picker it can honour - #: rather than offering an option the server will refuse (R4: business is admin-only). - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - "tenantWideTypes": list(TENANT_WIDE_TYPES)} - - -@router.post("/admin/keychain", status_code=201) -def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): - kc = _kc() - body = body or {} - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - scope = clean_scope(body.get("scope")) - if scope is None: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can create one. You can add it as a personal connection instead.") - etype = str(body.get("type") or "").strip().lower() - if scope == "personal" and etype in TENANT_WIDE_TYPES: - # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this - # credential is what the WHOLE workspace's databases are built from, so "personal" would - # be a label on a tenant-wide key rather than a boundary around it. - raise err(400, "scope_not_available", - f"a {etype} connection is what this whole workspace's databases are read " - f"through, so it is always business-wide — it cannot be a personal connection. " - f"An administrator can add it for everyone.") - try: - row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), - body.get("fields"), session.uname) - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", - f"the keychain is locked — {e}. A secret is never stored unencrypted.") - except ValueError as e: - raise err(400, "bad_entry", str(e)) - except Exception: - raise err(503, "store_unavailable", "the entry was not saved — try again") - # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode - # of a side bucket is to publish a credential, not to hide one. So the second write is not - # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was - # stored. `business` needs no row at all, so this branch is the only one that can be partial. - # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing - # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin - # find a second switch. The tables were never dropped, so "resume" is one flag. - if etype == "odoo": - try: - _rel_reconnect(session.runtime) - except Exception: # noqa: BLE001 - pass - if scope != DEFAULT_SCOPE: - try: - _write_scope(session.runtime, row["id"], scope, session.uname) - except Exception: - try: - kc.delete_entry(session.runtime, row["id"]) - except Exception: # noqa: BLE001 - pass - raise err(503, "store_unavailable", - "the key was not saved — its sharing setting could not be stored, so " - "nothing was kept. Try again.") - return {"entry": {**row, "scope": scope, "owner": session.uname}} - - -@router.put("/admin/keychain/{entry_id}") -def update_key(entry_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so - "edit this key" means "replace it" and that is `DELETE` + `POST`.""" - row = _own_row(session, entry_id) - scope = clean_scope((body or {}).get("scope"), default=None) - if scope is None: - raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can make one business-wide.") - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to change") - if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: - raise err(400, "scope_not_available", - f"a {row.get('type')} connection is what this whole workspace's databases are " - f"read through, so it is always business-wide.") - owner = row.get("owner") or session.uname - try: - _write_scope(session.runtime, entry_id, scope, owner) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} - - -@router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): - row = _own_row(session, entry_id) - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to delete") - try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} - - -@router.post("/admin/keychain/{entry_id}/test") -def test_key(entry_id: str, session: Session = Depends(require_session)): - _own_row(session, entry_id) # 404 for an entry this caller may not see - return _kc().test_entry(session.runtime, entry_id) - - -def _unsynced_customer_records(session): - """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no - longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a - tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the - answer is `known: False`, never a fabricated zero.""" - try: - import core.table_store as table_store - bucket = session.runtime.get("customer_table_workspace") or {} - overlay_pids = {} - for uname, ws in bucket.items(): - if uname == table_store.SHARED_KEY or not isinstance(ws, dict): - continue - for pid, cells in (ws.get("overlays") or {}).items(): - if isinstance(cells, dict) and cells: - overlay_pids.setdefault(str(pid), cells) - if not overlay_pids: - return {"known": True, "count": 0, "rows": []} - from routes_customers import allowed_pids - pool = {str(p) for p in allowed_pids(session)} - orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) - if str(x).isdigit() else 0) - rows = [] - for p in orphans[:50]: - cells = overlay_pids[p] - hint = next((str(v) for v in cells.values() if str(v).strip()), "") - rows.append({"pid": int(p) if str(p).isdigit() else p, - "fields": len(cells), "hint": hint[:80]}) - return {"known": True, "count": len(orphans), "rows": rows, - "shown": min(len(orphans), 50)} - except Exception as e: - return {"known": False, "count": None, "rows": [], - "note": f"pool unavailable — {type(e).__name__}"} - - -@router.get("/admin/connectors") -def connectors(session: Session = Depends(require_session)): - kc = _kc() - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same - # facts with a status column. Reading `kc.list_entries` here instead would have shown a member - # every colleague's personal connection on the screen next door to the one that hides them. - entries = visible_entries(session.runtime, session.uname, bool(session.admin)) - # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — - # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env - # for tenant #0 only, else nothing (fail closed — never another tenant's environment). - # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source - # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would - # make the answer depend on who opened the pane. Personal entries are excluded for the same - # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. - _scopes = _scope_rows(session.runtime) - first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) - if e["type"] == "odoo" - and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) - # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the - # connectors DIRECTORY answer "does the environment serve this tenant?" with one function - # instead of two spellings that agreed until a second tenant signed in. - if first_odoo and kc.unlocked(): - resolved = f"keychain:{first_odoo}" - elif env_odoo_available(session.runtime): - resolved = "env" - else: - resolved = None - rows = [] - if env_odoo_available(session.runtime): - rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", - "source": "env", "active": resolved == "env", - # the deployment's own credential — business-wide by construction, and it - # has no owner to be personal to. - "scope": DEFAULT_SCOPE, "owner": "", - "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) - for e in entries: - rows.append({"key": e["id"], "label": e["label"], "type": e["type"], - "source": "keychain", "preview": e["preview"], - "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", - "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), - "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) - out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. - # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the - # hole it left: "measures not already computed may still reach the source". D-10 - # closed that hole — `harness/datastore.py` (the mirror every measure column is - # answered from) refuses to sync while paused, and `routes_products._pool_for` got the - # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather - # than left standing, because a warning that outlives its defect teaches the reader to - # ignore warnings. - # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely - # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and - # collapsing them into "everything freezes" would be the kind of tidy summary that - # stops being true the first time one of them changes. - # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the - # correction is recorded here rather than silently applied. D-62 said this note - # "promises a behaviour on a dashboard measure path that has been dead since W16". - # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from - # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while - # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is - # `/api/v1/pages/{key}` (D-52), which this note never mentioned. - # - # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures - # stop moving rather than going blank". BOTH pool paths answer **503** when they have - # no copy to serve — the customer path for a scope with no snapshot - # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because - # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says - # so in as many words). So a paused connector plus a restarted server is exactly the - # blank screen this sentence promised could not happen. A warning that over-promises - # is worse than none: it is the sentence somebody quotes when the screen disagrees. - "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " - "views stay, and nothing reaches the source while it is paused. " - "Anything this server has already read keeps showing: the customer " - "workspace serves its pause-time snapshot, the product list serves " - "the last copy read since startup, and measure columns keep answering " - "from the mirror as it stood when you paused. What has NOT been read " - "cannot be shown — a scope with no snapshot, or the product list after " - "a restart, reports that the source is paused instead of showing " - "figures. Resume to start reading live again.")} - # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to - # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on - # the payload that is NOT about connectors: it unions overlays across EVERY user of the - # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user - # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member - # would have read colleagues' typed notes off the Connectors pane. It also filters against - # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and - # the number itself becomes wrong for them as well as private. - # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already - # returned, and the audit has to walk the payload, not the entry list I was thinking about. - if session.tenant == "royal-imports" and session.admin: - out["unsynced"] = _unsynced_customer_records(session) - return out - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database -# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to -# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a -# switch flipped here is a switch the sync path obeys — a config the route knows and the sync -# path does not is a control that does nothing and reports success. -def _rel(): - import odoo_relational as rel - return rel - - -def _odoo_entry(session): - """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing - is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" - kc = _kc() - scopes = _scope_rows(session.runtime) - for e in kc.list_entries(session.runtime): - if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": - return e - return None - - -def _odoo_source_fields(session, entry): - """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. - - ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no - route returns its output; this returns the three fields that identify WHICH server, and the - api key is not among them. The masked preview is the entry's own and was computed at write. - ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by - the process, and an admin editing it from a tenant screen would be editing the container. - """ - if entry is None: - return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), - os.environ.get("ODOO_USER", ""), False) - try: - f = _kc().read_fields(session.runtime, entry["id"]) or {} - except Exception: # noqa: BLE001 - return ("", "", "", True) # locked keychain: honest blanks, still editable - return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) - - -def _odoo_admin(session): - """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the - whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide - type cannot be personal in the first place.""" - if not session.admin: - raise err(403, "not_admin", - "the Odoo connection serves this whole workspace, so only an administrator can " - "configure it") - - -@router.get("/admin/connectors/odoo/config") -def odoo_config(session: Session = Depends(require_session)): - """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a - person needs to see before changing any of it.""" - _odoo_admin(session) - rel = _rel() - entry = _odoo_entry(session) - server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) - cfg = rel.read_config(session.runtime) - return { - "applicable": bool(rel.is_royal(session.tenant)), - "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) - else "none"), - "entryId": (entry or {}).get("id", ""), - "label": (entry or {}).get("label", "Odoo (environment)"), - "preview": (entry or {}).get("preview", ""), - "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, - "serverDbEditable": editable, - "grids": rel.grid_choices(session.runtime), - "syncEvery": cfg["syncEvery"], - "syncOptions": list(rel.SYNC_PRESETS), - "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, - "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], - # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: - # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders - # no button rather than one that 400s. - "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) - and not cfg["frozen"]), - } - - -@router.put("/admin/connectors/odoo/config") -def odoo_config_put(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED - back rather than silently applied.""" - _odoo_admin(session) - rel = _rel() - body = body or {} - notes = [] - - grids = body.get("grids") - known = {c["key"] for c in rel.grid_choices(session.runtime)} - clean_grids = None - if isinstance(grids, dict): - unknown = sorted(str(k) for k in grids if str(k) not in known) - if unknown: - # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid - # this connector does not have, and swallowing it makes the two disagree quietly. - raise err(400, "unknown_grid", - f"this connector has no grid called {', '.join(unknown)}") - clean_grids = {str(k): bool(v) for k, v in grids.items()} - if clean_grids and not any(clean_grids.get(k, True) for k in known): - notes.append("every grid is switched off — nothing will be materialised on the next " - "sync, and the databases you already have are left untouched") - - every = body.get("syncEvery") - clean_every = None - if every is not None: - clean_every = str(every).strip().lower() - if clean_every not in rel.SYNC_PRESETS: - # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. - # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, - # and never silently ignored either, because a control that discards your answer - # without a word is how a limit becomes invisible. - clean_every = rel.DEFAULT_SYNC - notes.append(f"{every!r} is not an interval this connector offers, and anything under " - f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " - f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") - - server_db = body.get("serverDb") - if server_db is not None: - server_db = " ".join(str(server_db).split())[:80] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if clean_grids is not None: - cur.setdefault("grids", {}).update(clean_grids) - if clean_every is not None: - cur["syncEvery"] = clean_every - return cur - - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - - if server_db: - notes.append(_rewrite_server_db(session, server_db)) - - # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE - # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: - # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( - # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So - # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value - # is stored, clamped, displayed and never read. - # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a - # default 1800 s and syncs anyway. "Only when I ask" asks all the same. - # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not - # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it - # is REPORTED here, at the moment of the change, with what it really controls. Delete these - # notes when the loop becomes per-tenant, not before. - if clean_every is not None: - if not rel.is_royal(session.tenant): - notes.append("this interval is saved, but the sync loop currently reads its schedule " - "from one workspace for the whole deployment — so it will not change how " - "often YOUR data refreshes until per-workspace scheduling ships") - else: - notes.append("this interval is saved and it is the one the deployment's sync loop " - "uses — it changes the refresh rate for every workspace on this " - "deployment, not only this one") - if clean_every == "manual": - notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " - "manual-only branch, so data still refreshes on the default interval") - - out = odoo_config(session) - return {**out, "notes": [n for n in notes if n]} - - -def _rewrite_server_db(session, server_db): - """Point the stored Odoo credential at a different server database (R9's first reading). - - ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of - how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. - So this is add-then-delete through the module's own doors, with the side rows (pause flag, - scope) carried across because they are keyed by ENTRY ID. - ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the - delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a - resync landing inside that window could read the OLD database. The alternative order can - leave the workspace with no credential at all, which is worse than one stale read. Milliseconds - of ambiguity beats a lost key. - """ - kc = _kc() - entry = _odoo_entry(session) - if entry is None: - return ("the server database is set on this deployment's environment, not in the " - "keychain, so it was not changed here") - try: - fields = kc.read_fields(session.runtime, entry["id"]) or {} - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", f"the keychain is locked — {e}") - if not fields: - raise err(400, "bad_entry", "this credential could not be read back to be changed") - if str(fields.get("db") or "") == server_db: - return "" - fields["db"] = server_db - try: - new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) - except Exception: - raise err(503, "store_unavailable", - "the server database was not changed — the existing connection is untouched") - # carry the side rows across, then retire the old entry - try: - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - if entry["id"] in flags: - def _mv(cur): - cur[new["id"]] = cur.pop(entry["id"], {}) - return cur - session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) - kc.delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - except Exception: # noqa: BLE001 - return (f"the connection now points at {server_db}, but the previous credential could " - f"not be removed — delete it under Keychains") - return f"the connection now points at the {server_db} database" - - -@router.post("/admin/connectors/odoo/disconnect") -def odoo_disconnect(session: Session = Depends(require_session)): - """R10 — remove the credential and FREEZE the grids as static data. - - ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the - key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — - including the boot rebuild and the resync loop, which for tenant #0 would otherwise - re-materialise from the process ENVIRONMENT and quietly undo the disconnect. - ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and - every FIELD DEFINITION stays, user-added columns included, because a field a person added is - the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes - one flag in a different bucket, which is what makes that guarantee structural rather than - careful. - """ - _odoo_admin(session) - rel = _rel() - import datetime as _dt - entry = _odoo_entry(session) - if not entry and not env_odoo_available(session.runtime): - raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = True - cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - cur["frozenBy"] = str(session.uname) - return cur - - # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT - # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which - # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, - # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read - # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, - # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected - # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's - # own sentence promised *"nothing is being refreshed"*. - # - # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause - # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` - # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed - # from `pause_connector` because it learned them the hard way: - # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no - # resolved source left to name, and the flag would land under a key nothing reads (D-10). - # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to - # serve; a failed capture leaves the connector live rather than paused-with-nothing. - _, flag_key = _resolved_odoo_key(session.runtime) - snapshots = 0 - if flag_key: - try: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - except Exception: # noqa: BLE001 - # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. - snapshots = 0 - - # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and - # the connector is still live; if the delete failed AFTER the flag landed, the tenant is - # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order - # can leave a workspace with no key and a connector that still tries to sync. - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "nothing was disconnected — try again") - - paused_mirror = False - if flag_key: - def _pause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": True} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") - paused_mirror = True - except Exception: # noqa: BLE001 - paused_mirror = False - - removed = "" - if entry is not None: - try: - _kc().delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - removed = entry["id"] - except Exception: - raise err(503, "store_unavailable", - "the databases are frozen but the stored credential was not removed — " - "delete it under Keychains") - - # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the - # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT - # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a - # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: - # the limit that cannot be removed is REPORTED, with what to do instead. - note = ("Your Odoo databases are frozen: every row and every column you had is still there and " - "still readable, and nothing is being refreshed.") - if not paused_mirror and flag_key: - note += (" ⚠ The live mirror could not be paused, so the two read-through databases " - "(order lines and GL lines) may keep advancing — pause the Odoo connector under " - "Keychains to stop them.") - note += (" Reconnecting adds the key back and resumes into the same databases." if entry - else " This workspace's Odoo credential comes from the deployment environment, so " - "there was no stored key to remove — the databases are frozen and Reconnect " - "resumes them into the same tables.") - return {"frozen": True, "removedEntry": removed, - # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean - # nobody returns is a guarantee nobody can check. - "pausedMirror": paused_mirror, "snapshots": snapshots, - "source": "keychain" if entry else "env", - "note": note} - - -@router.post("/admin/connectors/odoo/reconnect") -def odoo_reconnect(session: Session = Depends(require_session)): - """R10's second sentence — *"Reconnecting resumes into the same tables."* - - It clears the freeze and nothing else: the tables were never dropped, so there is nothing to - recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this - is the switch that lets the sync path see it again. - """ - _odoo_admin(session) - try: - _rel_reconnect(session.runtime) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only - # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the - # two read-through grids would sit at the disconnect date while the panel said "connected - # again" — the same disagreement between the halves of "frozen", pointing the other way. - # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the - # resolved source only exists again by this point. - _, flag_key = _resolved_odoo_key(session.runtime) - resumed = False - if flag_key: - def _unpause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": False} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") - resumed = True - except Exception: # noqa: BLE001 - resumed = False - connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) - return {"frozen": False, "connected": connected, "resumedMirror": resumed, - "note": ("Odoo is connected again and the databases you already had will refresh in " - "place." if connected else - "The freeze is lifted, but there is no Odoo credential yet — add one under " - "Keychains and the databases resume into the same tables.")} - - -@router.post("/admin/connectors/{key}/pause") -def pause_connector(key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - paused = bool((body or {}).get("paused")) - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an - # admin act; pausing your own personal one is yours. The env source has no keychain row and - # is business-wide by construction, hence the admin fallthrough. - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(key)), None) - if row is not None: - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to pause") - elif not session.admin: - raise err(403, "forbidden", "administrators only") - - # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a - # "last successful sync" to serve before the freeze takes effect. Capturing before the - # flag flips means a failed capture leaves the connector live (never paused-with-nothing). - snapshots = 0 - _, flag_key = _resolved_odoo_key(session.runtime) - if paused and flag_key and str(key) == flag_key: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - - def _up(cur): - cur[str(key)] = {"paused": paused} - return cur - - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"key": key, "paused": paused, "snapshots": snapshots} +"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). + +Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER +return a decrypted field — list rows carry a masked preview, and the decrypt function is a +connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's +env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** +count (rows holding overlay data whose pids the current pool no longer serves; counted and +drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly +in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with +the keychain cutover wave R3 staged. +""" +import os + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session +# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door +# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall +# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per +# entry instead of per endpoint. `session.admin` is still what business-wide requires. + +router = APIRouter(prefix="/api/v1") + +#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by +#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another +#: freezes nothing while reporting success, which is the shape of the bug D-10 books. +from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 + ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) +#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED +#: odoo connector is paused; read by the pool path while paused; survives a Space restart. +SNAPSHOT_KEY = "connector_snapshots" + +#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. +#: +#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves +#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and +#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's +#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's +#: connection, and handing it to another tenant is the leak this method exists to prevent."* +#: +#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and +#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value +#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, +#: the gate reds instead of the product quietly disagreeing with itself. +ENV_ODOO_TENANT = "royal-imports" + + +def env_odoo_available(rt): + """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) + + ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. + `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant + guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was + `connected` and offered "Manage keys" for a credential belonging to another company. This + module asked the same question correctly two functions below, which is the whole shape of + [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in + as the second tenant. + + ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which + source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, + wrong for "should the environment row be listed at all", which is what the connectors pane + needs in order to show an inactive env source beside an active keychain one. + + Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is + how another tenant's environment got reported as this tenant's connection in the first place. + """ + try: + if not (os.environ.get("ODOO_URL") or "").strip(): + return False + return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT + except Exception: # noqa: BLE001 + return False + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. +# Every keychain entry and connector carries a scope, selectable per connection; business-wide is +# admin-only and applies to every user in the tenant."* +# +# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file +# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is +# an access rule, not a secret, so it lives in the layer that already decides who may ask. +# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves +# cannot drift into two vocabularies. +SCOPES = ("business", "personal") +#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before +#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. +#: Materialising it would be a migration nobody authorised and would rewrite the store on the next +#: read [[a-migration-that-runs-on-the-next-write]]. +DEFAULT_SCOPE = "business" +#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. +#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean +#: the same thing and there is only one way to spell the common case. +SCOPE_KEY = "keychain_scopes" + +#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. +#: +#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* +#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is +#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they +#: would silently become the credential the entire workspace's databases are built from, which is +#: a credential elevation wearing a scope picker. +#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes +#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is +#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door +#: from the only side B owns, and the resolver-side guard is booked for A. +TENANT_WIDE_TYPES = ("odoo", "meta_ads") + + +def clean_scope(raw, default=DEFAULT_SCOPE): + """A scope word from the wire, or None when the caller said something we do not speak. + + Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole + reason this returns None rather than falling back: a typo'd `"personel"` silently becoming + business-wide is exactly the failure a scope picker exists to prevent. + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + got = str(raw).strip().lower() + return got if got in SCOPES else None + + +def _scope_rows(rt): + try: + return dict(rt.get(SCOPE_KEY) or {}) + except Exception: # noqa: BLE001 + return {} + + +def entry_scope(rt, entry_id, rows=None): + """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a + census does not re-read the store once per entry.""" + r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) + if not isinstance(r, dict): + return DEFAULT_SCOPE, "" + return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) + + +def may_see(scope, owner, uname): + """R4's visibility rule: business-wide is everyone's, personal is its owner's. + + ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the + per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token + is identity, not infrastructure"*). An admin who could read every member's personal credential + would make "personal" a label rather than a boundary. + """ + return scope != "personal" or str(owner) == str(uname) + + +def visible_entries(rt, uname, is_admin=False): + """This USER's view of the keychain: every business entry plus their own personal ones, each + row carrying its `scope` and `owner` so no caller has to ask a second time. + + ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can + hold a connection of their own; it does not hand them four characters of the workspace's Odoo + key. So a business row a member did not create arrives WITHOUT `preview` — they can see that + the connection exists and is theirs to use, which is the whole of what R4 grants. The default + is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. + """ + rows = _scope_rows(rt) + out = [] + for e in _kc().list_entries(rt): + scope, owner = entry_scope(rt, e["id"], rows) + if not owner: + owner = str(e.get("createdBy") or "") + if not may_see(scope, owner, uname): + continue + row = {**e, "scope": scope, "owner": owner} + if not (is_admin or str(owner) == str(uname)): + row["preview"] = "" + out.append(row) + return out + + +def _write_scope(rt, entry_id, scope, owner): + """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the + default, so the store holds one spelling of the common case.""" + def _up(cur): + if scope == DEFAULT_SCOPE: + cur.pop(str(entry_id), None) + else: + cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} + return cur + + rt.update(SCOPE_KEY, _up, flush="sync") + return True + + +def _may_touch(session, row): + """May this session change or delete `row`? An admin owns the business-wide ones; a member + owns their own personal ones. Anything else is not theirs to move.""" + if row.get("scope") == "personal": + return str(row.get("owner") or "") == str(session.uname) + return bool(session.admin) + + +def _kc(): + import core.keychain as keychain + return keychain + + +def _resolved_odoo_key(rt): + """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the + shapes this module needs: the display string (`env` / `keychain:`) and the key the pause + flag is stored under. + + ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, + beside the `odoo_source()` it must agree with. This function had its own copy of the same + three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and + a second copy is exactly how a pause flag comes to be written against one resolution and read + against another, freezing nothing while the UI reports success. The two SHAPES stay here + because they are this module's presentation concern; the DECISION does not. + """ + flag_key = rt.odoo_flag_key() + if not flag_key: + return None, None + return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key + + +def odoo_paused(rt): + """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry + that is not the resolved source freezes nothing — it serves nothing. + + ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the + measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question + the customer pool does. This name stays because `routes_customers`, `routes_products` and + `verify_api` all call it — moving the logic without moving the door keeps one answer and + costs no caller a change. + """ + return bool(rt.odoo_paused()) + + +def _snap_scope_key(team_id, agent): + return f"t={team_id}|a={agent}" + + +def load_pool_snapshot(rt, team_id, agent): + """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider + scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" + try: + snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} + e = snap.get(_snap_scope_key(team_id, agent)) + if isinstance(e, dict) and isinstance(e.get("rows"), list): + return float(e.get("ts") or 0), e["rows"] + except Exception: + pass + return None + + +def save_pool_snapshots(rt, taken_by=""): + """Persist every currently-cached pool scope as the pause-time snapshot ('the last + successful sync', made concrete). Ensures the consolidated default scope exists first so + a pause on a cold process still captures something to serve.""" + import time as _time + import routes_customers as _rc + try: + _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on + except Exception: + pass # cold + Odoo down: persist whatever IS cached + pools = {} + for key, entry in list(rt.pool_cache.items()): + if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" + and isinstance(entry, tuple) and len(entry) == 2 + and isinstance(entry[1], list)): + pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} + if not pools: + return 0 + + def _up(cur): + cur["odoo_pool"] = pools + cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") + cur["takenBy"] = str(taken_by or "") + return cur + + rt.update(SNAPSHOT_KEY, _up, flush="sync") + return len(pools) + + +def _rel_reconnect(rt): + """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot + disagree about what "resume" means.""" + import odoo_relational as rel + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = False + cur.pop("frozenAt", None) + cur.pop("frozenBy", None) + return cur + + rt.update(rel.CONFIG_KEY, _up, flush="sync") + return True + + +# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION +# +# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as +# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so +# the risk is what this block is mostly about. +# +# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env +# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already +# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, +# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build +# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks +# empty while its Odoo grids work. +# +# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns +# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — +# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under +# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector +# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That +# is D-259's exact shape (a pause written under one key and read under another) and +# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the +# SAME pass, and the carry is asserted. + + +def _env_odoo_fields(): + """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. + + ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. + Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a + PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo + offline, on a deployment where it had been working. The env fallback cannot save it, because the + entry's existence is what turns the fallback off. + ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the + field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two + vocabularies meet here; nowhere else. + """ + want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), + ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) + got = {field: (os.environ.get(env) or "").strip() for field, env in want} + missing = sorted(env for field, env in want if not got[field]) + if missing: + return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " + f"would take this tenant's Odoo offline rather than migrate it") + return got, "" + + +def migrate_env_odoo(rt): + """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. + + `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every + path including the skips, because "already migrated", "no keychain key on this deployment" and + "the env is incomplete" are three different operator actions. + + ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success + every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here + would be a dry run that lies, and the thing it would lie about is a credential. + + ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a + third-party failure must not take the boot down. + """ + out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + if not env_odoo_available(rt): + # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. + out["why"] = "this tenant has no environment Odoo credential to migrate" + return out + fields, why = _env_odoo_fields() + if not fields: + out["why"] = why + return out + # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the + # NEW key and the old one is unreachable through the resolver — so the only moment this fact can + # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. + try: + was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) + .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) + except Exception: # noqa: BLE001 + was_paused = False + row, why = _kc().ensure_entry_of_type( + rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") + if row is None: + out["why"] = why + return out + out["done"], out["entry"] = True, row["id"] + if was_paused: + # The freeze followed the credential. Without this the connector silently RESUMES pulling + # live Odoo at the first boot after the migration. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = dict(cur.get(row["id"]) or {}) + entry["paused"] = True + entry["pausedBy"] = "system" + entry["pausedNote"] = ("carried over from the environment source when the credential " + "was migrated onto the keychain") + cur[row["id"]] = entry + return cur + + try: + rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + out["carried_pause"] = True + except Exception as exc: # noqa: BLE001 + # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse + # than one that did not run, so this is the one failure that must never be silent. + out["why"] = (f"the credential migrated but the PAUSE could not be carried over " + f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + return out + + +def _own_row(session, entry_id): + """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the + caller cannot see: telling a member that somebody else's personal credential EXISTS is the + disclosure the scope is for.""" + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(entry_id)), None) + if row is None: + raise err(404, "no_entry", "no such key") + return row + + +@router.get("/admin/keychain") +def list_keychain(session: Session = Depends(require_session)): + """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. + R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would + ship the feature and no door to it. The wall moved INTO the payload: a member sees the + business-wide entries and their own, never anybody else's personal one.""" + kc = _kc() + return {"entries": visible_entries(session.runtime, session.uname, + bool(session.admin)), + "locked": not kc.unlocked(), + #: the vocabulary and the permission, so the client renders a picker it can honour + #: rather than offering an option the server will refuse (R4: business is admin-only). + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + "tenantWideTypes": list(TENANT_WIDE_TYPES)} + + +@router.post("/admin/keychain", status_code=201) +def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): + kc = _kc() + body = body or {} + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + scope = clean_scope(body.get("scope")) + if scope is None: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can create one. You can add it as a personal connection instead.") + etype = str(body.get("type") or "").strip().lower() + if scope == "personal" and etype in TENANT_WIDE_TYPES: + # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this + # credential is what the WHOLE workspace's databases are built from, so "personal" would + # be a label on a tenant-wide key rather than a boundary around it. + raise err(400, "scope_not_available", + f"a {etype} connection is what this whole workspace's databases are read " + f"through, so it is always business-wide — it cannot be a personal connection. " + f"An administrator can add it for everyone.") + try: + row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), + body.get("fields"), session.uname) + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", + f"the keychain is locked — {e}. A secret is never stored unencrypted.") + except ValueError as e: + raise err(400, "bad_entry", str(e)) + except Exception: + raise err(503, "store_unavailable", "the entry was not saved — try again") + # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode + # of a side bucket is to publish a credential, not to hide one. So the second write is not + # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was + # stored. `business` needs no row at all, so this branch is the only one that can be partial. + # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing + # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin + # find a second switch. The tables were never dropped, so "resume" is one flag. + if etype == "odoo": + try: + _rel_reconnect(session.runtime) + except Exception: # noqa: BLE001 + pass + if scope != DEFAULT_SCOPE: + try: + _write_scope(session.runtime, row["id"], scope, session.uname) + except Exception: + try: + kc.delete_entry(session.runtime, row["id"]) + except Exception: # noqa: BLE001 + pass + raise err(503, "store_unavailable", + "the key was not saved — its sharing setting could not be stored, so " + "nothing was kept. Try again.") + return {"entry": {**row, "scope": scope, "owner": session.uname}} + + +@router.put("/admin/keychain/{entry_id}") +def update_key(entry_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so + "edit this key" means "replace it" and that is `DELETE` + `POST`.""" + row = _own_row(session, entry_id) + scope = clean_scope((body or {}).get("scope"), default=None) + if scope is None: + raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can make one business-wide.") + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to change") + if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: + raise err(400, "scope_not_available", + f"a {row.get('type')} connection is what this whole workspace's databases are " + f"read through, so it is always business-wide.") + owner = row.get("owner") or session.uname + try: + _write_scope(session.runtime, entry_id, scope, owner) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} + + +@router.delete("/admin/keychain/{entry_id}") +def delete_key(entry_id: str, session: Session = Depends(require_session)): + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + try: + _kc().delete_entry(session.runtime, entry_id) + _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + return {"ok": True} + + +@router.post("/admin/keychain/{entry_id}/test") +def test_key(entry_id: str, session: Session = Depends(require_session)): + _own_row(session, entry_id) # 404 for an entry this caller may not see + return _kc().test_entry(session.runtime, entry_id) + + +def _unsynced_customer_records(session): + """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no + longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a + tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the + answer is `known: False`, never a fabricated zero.""" + try: + import core.table_store as table_store + bucket = session.runtime.get("customer_table_workspace") or {} + overlay_pids = {} + for uname, ws in bucket.items(): + if uname == table_store.SHARED_KEY or not isinstance(ws, dict): + continue + for pid, cells in (ws.get("overlays") or {}).items(): + if isinstance(cells, dict) and cells: + overlay_pids.setdefault(str(pid), cells) + if not overlay_pids: + return {"known": True, "count": 0, "rows": []} + from routes_customers import allowed_pids + pool = {str(p) for p in allowed_pids(session)} + orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) + if str(x).isdigit() else 0) + rows = [] + for p in orphans[:50]: + cells = overlay_pids[p] + hint = next((str(v) for v in cells.values() if str(v).strip()), "") + rows.append({"pid": int(p) if str(p).isdigit() else p, + "fields": len(cells), "hint": hint[:80]}) + return {"known": True, "count": len(orphans), "rows": rows, + "shown": min(len(orphans), 50)} + except Exception as e: + return {"known": False, "count": None, "rows": [], + "note": f"pool unavailable — {type(e).__name__}"} + + +@router.get("/admin/connectors") +def connectors(session: Session = Depends(require_session)): + kc = _kc() + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same + # facts with a status column. Reading `kc.list_entries` here instead would have shown a member + # every colleague's personal connection on the screen next door to the one that hides them. + entries = visible_entries(session.runtime, session.uname, bool(session.admin)) + # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — + # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env + # for tenant #0 only, else nothing (fail closed — never another tenant's environment). + # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source + # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would + # make the answer depend on who opened the pane. Personal entries are excluded for the same + # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. + _scopes = _scope_rows(session.runtime) + first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) + if e["type"] == "odoo" + and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) + # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the + # connectors DIRECTORY answer "does the environment serve this tenant?" with one function + # instead of two spellings that agreed until a second tenant signed in. + if first_odoo and kc.unlocked(): + resolved = f"keychain:{first_odoo}" + elif env_odoo_available(session.runtime): + resolved = "env" + else: + resolved = None + rows = [] + if env_odoo_available(session.runtime): + rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", + "source": "env", "active": resolved == "env", + # the deployment's own credential — business-wide by construction, and it + # has no owner to be personal to. + "scope": DEFAULT_SCOPE, "owner": "", + "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) + for e in entries: + rows.append({"key": e["id"], "label": e["label"], "type": e["type"], + "source": "keychain", "preview": e["preview"], + "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", + "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), + "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) + out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. + # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the + # hole it left: "measures not already computed may still reach the source". D-10 + # closed that hole — `harness/datastore.py` (the mirror every measure column is + # answered from) refuses to sync while paused, and `routes_products._pool_for` got the + # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather + # than left standing, because a warning that outlives its defect teaches the reader to + # ignore warnings. + # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely + # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and + # collapsing them into "everything freezes" would be the kind of tidy summary that + # stops being true the first time one of them changes. + # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the + # correction is recorded here rather than silently applied. D-62 said this note + # "promises a behaviour on a dashboard measure path that has been dead since W16". + # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from + # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while + # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is + # `/api/v1/pages/{key}` (D-52), which this note never mentioned. + # + # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures + # stop moving rather than going blank". BOTH pool paths answer **503** when they have + # no copy to serve — the customer path for a scope with no snapshot + # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because + # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says + # so in as many words). So a paused connector plus a restarted server is exactly the + # blank screen this sentence promised could not happen. A warning that over-promises + # is worse than none: it is the sentence somebody quotes when the screen disagrees. + "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " + "views stay, and nothing reaches the source while it is paused. " + "Anything this server has already read keeps showing: the customer " + "workspace serves its pause-time snapshot, the product list serves " + "the last copy read since startup, and measure columns keep answering " + "from the mirror as it stood when you paused. What has NOT been read " + "cannot be shown — a scope with no snapshot, or the product list after " + "a restart, reports that the source is paused instead of showing " + "figures. Resume to start reading live again.")} + # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to + # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on + # the payload that is NOT about connectors: it unions overlays across EVERY user of the + # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user + # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member + # would have read colleagues' typed notes off the Connectors pane. It also filters against + # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and + # the number itself becomes wrong for them as well as private. + # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already + # returned, and the audit has to walk the payload, not the entry list I was thinking about. + if session.tenant == "royal-imports" and session.admin: + out["unsynced"] = _unsynced_customer_records(session) + return out + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database +# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to +# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a +# switch flipped here is a switch the sync path obeys — a config the route knows and the sync +# path does not is a control that does nothing and reports success. +def _rel(): + import odoo_relational as rel + return rel + + +def _odoo_entry(session): + """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing + is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" + kc = _kc() + scopes = _scope_rows(session.runtime) + for e in kc.list_entries(session.runtime): + if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": + return e + return None + + +def _odoo_source_fields(session, entry): + """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. + + ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no + route returns its output; this returns the three fields that identify WHICH server, and the + api key is not among them. The masked preview is the entry's own and was computed at write. + ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by + the process, and an admin editing it from a tenant screen would be editing the container. + """ + if entry is None: + return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), + os.environ.get("ODOO_USER", ""), False) + try: + f = _kc().read_fields(session.runtime, entry["id"]) or {} + except Exception: # noqa: BLE001 + return ("", "", "", True) # locked keychain: honest blanks, still editable + return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) + + +def _odoo_admin(session): + """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the + whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide + type cannot be personal in the first place.""" + if not session.admin: + raise err(403, "not_admin", + "the Odoo connection serves this whole workspace, so only an administrator can " + "configure it") + + +@router.get("/admin/connectors/odoo/config") +def odoo_config(session: Session = Depends(require_session)): + """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a + person needs to see before changing any of it.""" + _odoo_admin(session) + rel = _rel() + entry = _odoo_entry(session) + server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) + cfg = rel.read_config(session.runtime) + return { + "applicable": bool(rel.is_royal(session.tenant)), + "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) + else "none"), + "entryId": (entry or {}).get("id", ""), + "label": (entry or {}).get("label", "Odoo (environment)"), + "preview": (entry or {}).get("preview", ""), + "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, + "serverDbEditable": editable, + "grids": rel.grid_choices(session.runtime), + "syncEvery": cfg["syncEvery"], + "syncOptions": list(rel.SYNC_PRESETS), + "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, + "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], + # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: + # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders + # no button rather than one that 400s. + "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) + and not cfg["frozen"]), + } + + +@router.put("/admin/connectors/odoo/config") +def odoo_config_put(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED + back rather than silently applied.""" + _odoo_admin(session) + rel = _rel() + body = body or {} + notes = [] + + grids = body.get("grids") + known = {c["key"] for c in rel.grid_choices(session.runtime)} + clean_grids = None + if isinstance(grids, dict): + unknown = sorted(str(k) for k in grids if str(k) not in known) + if unknown: + # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid + # this connector does not have, and swallowing it makes the two disagree quietly. + raise err(400, "unknown_grid", + f"this connector has no grid called {', '.join(unknown)}") + clean_grids = {str(k): bool(v) for k, v in grids.items()} + if clean_grids and not any(clean_grids.get(k, True) for k in known): + notes.append("every grid is switched off — nothing will be materialised on the next " + "sync, and the databases you already have are left untouched") + + every = body.get("syncEvery") + clean_every = None + if every is not None: + clean_every = str(every).strip().lower() + if clean_every not in rel.SYNC_PRESETS: + # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. + # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, + # and never silently ignored either, because a control that discards your answer + # without a word is how a limit becomes invisible. + clean_every = rel.DEFAULT_SYNC + notes.append(f"{every!r} is not an interval this connector offers, and anything under " + f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " + f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") + + server_db = body.get("serverDb") + if server_db is not None: + server_db = " ".join(str(server_db).split())[:80] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if clean_grids is not None: + cur.setdefault("grids", {}).update(clean_grids) + if clean_every is not None: + cur["syncEvery"] = clean_every + return cur + + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + + if server_db: + notes.append(_rewrite_server_db(session, server_db)) + + # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE + # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: + # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( + # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So + # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value + # is stored, clamped, displayed and never read. + # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a + # default 1800 s and syncs anyway. "Only when I ask" asks all the same. + # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not + # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it + # is REPORTED here, at the moment of the change, with what it really controls. Delete these + # notes when the loop becomes per-tenant, not before. + if clean_every is not None: + if not rel.is_royal(session.tenant): + notes.append("this interval is saved, but the sync loop currently reads its schedule " + "from one workspace for the whole deployment — so it will not change how " + "often YOUR data refreshes until per-workspace scheduling ships") + else: + notes.append("this interval is saved and it is the one the deployment's sync loop " + "uses — it changes the refresh rate for every workspace on this " + "deployment, not only this one") + if clean_every == "manual": + notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " + "manual-only branch, so data still refreshes on the default interval") + + out = odoo_config(session) + return {**out, "notes": [n for n in notes if n]} + + +def _rewrite_server_db(session, server_db): + """Point the stored Odoo credential at a different server database (R9's first reading). + + ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of + how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. + So this is add-then-delete through the module's own doors, with the side rows (pause flag, + scope) carried across because they are keyed by ENTRY ID. + ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the + delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a + resync landing inside that window could read the OLD database. The alternative order can + leave the workspace with no credential at all, which is worse than one stale read. Milliseconds + of ambiguity beats a lost key. + """ + kc = _kc() + entry = _odoo_entry(session) + if entry is None: + return ("the server database is set on this deployment's environment, not in the " + "keychain, so it was not changed here") + try: + fields = kc.read_fields(session.runtime, entry["id"]) or {} + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", f"the keychain is locked — {e}") + if not fields: + raise err(400, "bad_entry", "this credential could not be read back to be changed") + if str(fields.get("db") or "") == server_db: + return "" + fields["db"] = server_db + try: + new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) + except Exception: + raise err(503, "store_unavailable", + "the server database was not changed — the existing connection is untouched") + # carry the side rows across, then retire the old entry + try: + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + if entry["id"] in flags: + def _mv(cur): + cur[new["id"]] = cur.pop(entry["id"], {}) + return cur + session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) + kc.delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + except Exception: # noqa: BLE001 + return (f"the connection now points at {server_db}, but the previous credential could " + f"not be removed — delete it under Keychains") + return f"the connection now points at the {server_db} database" + + +@router.post("/admin/connectors/odoo/disconnect") +def odoo_disconnect(session: Session = Depends(require_session)): + """R10 — remove the credential and FREEZE the grids as static data. + + ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the + key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — + including the boot rebuild and the resync loop, which for tenant #0 would otherwise + re-materialise from the process ENVIRONMENT and quietly undo the disconnect. + ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and + every FIELD DEFINITION stays, user-added columns included, because a field a person added is + the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes + one flag in a different bucket, which is what makes that guarantee structural rather than + careful. + """ + _odoo_admin(session) + rel = _rel() + import datetime as _dt + entry = _odoo_entry(session) + if not entry and not env_odoo_available(session.runtime): + raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = True + cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + cur["frozenBy"] = str(session.uname) + return cur + + # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT + # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which + # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, + # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read + # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, + # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected + # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's + # own sentence promised *"nothing is being refreshed"*. + # + # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause + # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` + # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed + # from `pause_connector` because it learned them the hard way: + # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no + # resolved source left to name, and the flag would land under a key nothing reads (D-10). + # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to + # serve; a failed capture leaves the connector live rather than paused-with-nothing. + _, flag_key = _resolved_odoo_key(session.runtime) + snapshots = 0 + if flag_key: + try: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + except Exception: # noqa: BLE001 + # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. + snapshots = 0 + + # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and + # the connector is still live; if the delete failed AFTER the flag landed, the tenant is + # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order + # can leave a workspace with no key and a connector that still tries to sync. + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "nothing was disconnected — try again") + + paused_mirror = False + if flag_key: + def _pause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": True} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") + paused_mirror = True + except Exception: # noqa: BLE001 + paused_mirror = False + + removed = "" + if entry is not None: + try: + _kc().delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + removed = entry["id"] + except Exception: + raise err(503, "store_unavailable", + "the databases are frozen but the stored credential was not removed — " + "delete it under Keychains") + + # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the + # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT + # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a + # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: + # the limit that cannot be removed is REPORTED, with what to do instead. + note = ("Your Odoo databases are frozen: every row and every column you had is still there and " + "still readable, and nothing is being refreshed.") + if not paused_mirror and flag_key: + note += (" ⚠ The live mirror could not be paused, so the two read-through databases " + "(order lines and GL lines) may keep advancing — pause the Odoo connector under " + "Keychains to stop them.") + note += (" Reconnecting adds the key back and resumes into the same databases." if entry + else " This workspace's Odoo credential comes from the deployment environment, so " + "there was no stored key to remove — the databases are frozen and Reconnect " + "resumes them into the same tables.") + return {"frozen": True, "removedEntry": removed, + # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean + # nobody returns is a guarantee nobody can check. + "pausedMirror": paused_mirror, "snapshots": snapshots, + "source": "keychain" if entry else "env", + "note": note} + + +@router.post("/admin/connectors/odoo/reconnect") +def odoo_reconnect(session: Session = Depends(require_session)): + """R10's second sentence — *"Reconnecting resumes into the same tables."* + + It clears the freeze and nothing else: the tables were never dropped, so there is nothing to + recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this + is the switch that lets the sync path see it again. + """ + _odoo_admin(session) + try: + _rel_reconnect(session.runtime) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only + # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the + # two read-through grids would sit at the disconnect date while the panel said "connected + # again" — the same disagreement between the halves of "frozen", pointing the other way. + # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the + # resolved source only exists again by this point. + _, flag_key = _resolved_odoo_key(session.runtime) + resumed = False + if flag_key: + def _unpause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": False} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") + resumed = True + except Exception: # noqa: BLE001 + resumed = False + connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) + return {"frozen": False, "connected": connected, "resumedMirror": resumed, + "note": ("Odoo is connected again and the databases you already had will refresh in " + "place." if connected else + "The freeze is lifted, but there is no Odoo credential yet — add one under " + "Keychains and the databases resume into the same tables.")} + + +@router.post("/admin/connectors/{key}/pause") +def pause_connector(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + paused = bool((body or {}).get("paused")) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an + # admin act; pausing your own personal one is yours. The env source has no keychain row and + # is business-wide by construction, hence the admin fallthrough. + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(key)), None) + if row is not None: + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to pause") + elif not session.admin: + raise err(403, "forbidden", "administrators only") + + # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a + # "last successful sync" to serve before the freeze takes effect. Capturing before the + # flag flips means a failed capture leaves the connector live (never paused-with-nothing). + snapshots = 0 + _, flag_key = _resolved_odoo_key(session.runtime) + if paused and flag_key and str(key) == flag_key: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + + def _up(cur): + cur[str(key)] = {"paused": paused} + return cur + + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"key": key, "paused": paused, "snapshots": snapshots} diff --git a/api/routes_nav.py b/api/routes_nav.py index 8ac00d2f5aa6453e9d49d448a09672bd0bde8a23..9ae358b988e5faa26ed55d6b1e25fbb3aaba7bf3 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -1,890 +1,890 @@ -"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. - -SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who -may see what — a nav that hides a link the API would still serve is a UI courtesy, not a -permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), -so the nav and the 403 can never disagree about a grant. - -The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no -`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the -`prefs.json` Library preference is deliberately NOT applied — is documented on -`core.perms.nav_pages`, which is where it belongs: one place, both callers. -""" -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, perms, require_session - -router = APIRouter(prefix="/api/v1") - -#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder -#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a -#: surface (the server-filtered nav still decides what exists). -_NAV_PREFS_KEY = "nav_prefs" -_MAX_NAV_FOLDERS = 16 - -#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. -#: -#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a -#: separate bucket rather than another field in that one. Folders and placement are one -#: person's arrangement of their own rail — per-user by definition. What a database is CALLED -#: and what it looks like are facts about the database: a workspace where two people call the -#: same table different things has no shared vocabulary left to discuss it in. Same store, two -#: buckets, because they answer to two different owners. -#: -#: Shape: {"": {"icon": {"shape": , "tone": }, -#: "name": ""}} -_NAV_META_KEY = "nav_meta" - -#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these -#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, -#: so it is the one place the list is written twice. -#: -#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by -#: shape and maps the result, so a shape this whitelist let through and the renderer does not -#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the -#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of -#: that. If the two lists ever drift the symptom is an icon that silently reverts to the -#: default, which is the loudest SAFE failure available here. -_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", - "chart", "map", "users", "clock", "heart", "bolt"}) -_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) -_MAX_NAV_NAME = 60 - -#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. -#: -#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is -#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature -#: rather than a convenience one. -#: -#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. -#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An -#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one -#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls -#: off the list while forty slots hold repeat visits to the first. Keying by page makes -#: "recent" mean what the word means, and makes the cap bound the number of DATABASES -#: remembered rather than the number of clicks. -_NAV_RECENTS_KEY = "nav_recents" -_MAX_RECENTS = 50 - -#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a -#: precedent, not a preference. `user_tables.create` writes -#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses -#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 -#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — -#: with nothing to go red, because both ends are internally consistent. An integer instant has -#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect -#: shipped; this is that lesson applied before it.) -def _now() -> int: - return int(time.time()) - - -def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): - """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. - - `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown - or invisible key is dropped (it can return when the grant does — placement is cosmetic, - so pruning is loss-free). Unknown folder refs drop the placement, not the folder. - - `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the - `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than - pruned: keeping a placement whose database may since have been deleted is cosmetically - harmless, while pruning a live one is the silent data loss this function just stopped - causing. Never widened to non-`ut_` keys — those come from the compiled registry, which - cannot fail to enumerate. - """ - raw = raw if isinstance(raw, dict) else {} - folders, seen = [], set() - for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:40] - name = " ".join(str(f.get("name") or "").split())[:40] - if not fid or fid in seen or not name: - continue - seen.add(fid) - folders.append({"id": fid, "name": name}) - ids = {f["id"] for f in folders} - placement = {} - src = raw.get("placement") - if isinstance(src, dict): - for k, v in src.items(): - k, v = str(k)[:60], str(v)[:40] - if v not in ids: - continue # the folder is gone; the placement goes with it - if page_keys is None or k in page_keys: - placement[k] = v - elif keep_unknown_ut and k.startswith("ut_"): - placement[k] = v - return {"folders": folders, "placement": placement} - - -def _placeable_top_keys(session): - """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. - - ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by - construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and - `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's - user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. - So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement - naming one — on READ and on WRITE. - - The symptom was "a database I drag into a folder falls out of it later", never "it does not - work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy - (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. - The next reload served the pruned copy and the database was back at root. Built-in modules - (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly - why it read as intermittent rather than as a missing feature. - - ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents - block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the - ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 - fix did not reach. - - `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set - as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement - because the store blinked would be the original defect wearing a different cause. - """ - pages = perms.nav_pages(session.user) or [] - keys = {p["key"] for p in pages if not p.get("parent")} - try: - import core.user_tables as user_tables - # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never - # name a database this session cannot see, and the nav and this door agree by sharing - # one predicate rather than by two lists being kept in step. - # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is - # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires - # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured - # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the - # owner reports as "the Automation row arrives ten seconds late". - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - keys.add(str(e.get("key") or "")) - except Exception: - return keys, False - return keys, True - - -def _clean_icon(raw): - """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" - if not isinstance(raw, dict): - return None - shape, tone = raw.get("shape"), raw.get("tone") - if shape not in _ICON_SHAPES or tone not in _ICON_TONES: - return None - return {"shape": shape, "tone": tone} - - -def _read_nav_meta(runtime): - """The tenant's `nav_meta`, validated on the way OUT as well as in. - - Re-validating a read looks redundant and is not: the bucket outlives this code, an older - build may have written a shape this one no longer accepts, and the whitelist is mirrored - from a vocabulary that lives in another language. A row whose icon does not survive - validation renders the default mark — never a crash, and never a half-drawn glyph. - """ - try: - stored = runtime.get(_NAV_META_KEY) or {} - except Exception: - return {} # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return {} - out = {} - for key, meta in stored.items(): - if not isinstance(meta, dict): - continue - entry = {} - icon = _clean_icon(meta.get("icon")) - if icon: - entry["icon"] = icon - # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write - # route refuses a name for a built-in key, but a record written by an older build (or - # by hand) could still carry one — and honouring it would let the store rename - # `customer_data` on screen while `core/registry.py` and every other reader went on - # calling it Customer. A name that could not be written today is not served today. - name = meta.get("name") - if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): - entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] - if entry: - out[str(key)] = entry - return out - - -def _read_recents(runtime, uname, allowed): - """This user's recents, newest first, PRUNED to what they may currently see. - - ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: - - · the WRITE happens on every route open — it is the one hot path this file has — so it - must not build the whole nav to validate one key; - · a key whose grant was REVOKED must stop being offered without anyone running a - migration, and must come back if the grant does. That is `_clean_nav_prefs`' own - prune-never-invent posture, applied to a second bucket for the same reason. - - A key that is not in `allowed` therefore reaches the store and never reaches a screen — - which also means a stuffed key cannot be used to discover what exists: it comes back only - if the session could already see it. - """ - try: - stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} - except Exception: - return [] # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return [] - out = [] - for key, at in stored.items(): - key = str(key) - if allowed is not None and key not in allowed: - continue - try: - at = int(at) - except (TypeError, ValueError): - continue # a stamp this build cannot read is not a stamp - if at <= 0: - continue - out.append({"key": key, "at": at}) - out.sort(key=lambda r: r["at"], reverse=True) - return out[:_MAX_RECENTS] - - -# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── -# -# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the -# block that computed them (`_important_counts`) went with them. It read one store bucket PER -# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database -# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on -# **14** pages. It is **0 and 0** now. -# -# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the -# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold -# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. -# A budget that can be exhausted is a number that can be silently short, which is why it also had to -# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to -# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge -# instead of a late rail. Closes D-288 and D-289. -# -# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the -# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store -# read was — and duplicating them into the new route would have put two answers behind one badge. - -#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the -#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> -#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is -#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second -#: copy here would be free to drift from it ([[one-question-two-normalizers]]). -_WS_SUFFIX = "_table_workspace" - - -def _visible_views(doc, uname, is_admin, granted_ids): - """Every view on ONE database that THIS caller can see, from an already-read workspace doc. - - ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get - wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` - — one home per view, never both — so "how many views are marked important here" has a - DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and - `admin` has none, on the same database. A tenant-wide count would put a number in the owner's - rail that no view sidebar they can open would ever add up to. - - ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for - it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole - bucket per database (the N whole reads this block exists to avoid), and the only other option is - a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told - exists; a private import cannot. - """ - import core.table_store as table_store - out = {} - for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): - if isinstance(view, dict): - out[str(vid)] = view - for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): - if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): - out.setdefault(str(vid), view) - # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the - # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. - if granted_ids: - for stratum, blob in doc.items(): - if stratum == uname or not isinstance(blob, dict): - continue - for vid, view in (blob.get("views") or {}).items(): - if str(vid) in granted_ids and isinstance(view, dict): - out.setdefault(str(vid), view) - return out - - -def _view_record_count(cfg, cohorts): - """How many RECORDS this view resolves to, or None when that cannot be answered for free. - - ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact - because the view CARRIES its row set; the third — an ordinary filtered view — can only be - counted by running its filters over the records, and the records are the one thing this route - must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` - here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and - the database's `partial` flag says so, which is the whole of R6's second sentence applied to a - badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a - number that is short by an unknown amount. - - ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` - counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); - this end is the exact inverse — it has no rows at all and no window either. The two are honest - about different halves, which is why `partial` had to be on the wire rather than derived. - """ - if not isinstance(cfg, dict): - return None - # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` - # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. - lock = str(cfg.get("cohortLock") or "").strip() - if lock: - n = cohorts.get(lock) - return int(n) if isinstance(n, int) else None - # A curated row set carries its own count. - pids = cfg.get("memberPids") - if isinstance(pids, list) and pids: - return len(pids) - return None - - -def _visible_database_keys(session): - """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. - - ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: - - · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT - CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. - A star scan cannot borrow that: it would open the view bucket of a database this workspace's - catalogue does not include. `nav()` applies the catalogue filter; that helper never has. - · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so - it cannot be reached from another route without rebuilding the payload. - - So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same - order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the - tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes - the answer honest for a caller about to read a view bucket: a module surface with no workspace - (`sales`, `ar`) is not a database and has no views to star. - - ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` - excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: - a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even - though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious - reading — this ticket's first draft assumed the clause was excluding children and wrote a - docstring around a defect that does not exist. - - ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as - authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. - """ - import core.view_templates as view_templates - keys, enumerated = set(), True - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} - for p in (perms.nav_pages(session.user) or []): - key = str(p.get("key") or "") - if allowed is not None and key not in allowed \ - and str(p.get("parent") or "") not in allowed: - continue - if view_templates.workspace_key(key): - keys.add(key) - try: - import core.user_tables as user_tables - # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection - # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - k = str(e.get("key") or "") - if k: - keys.add(k) - except Exception: - enumerated = False - return keys, enumerated - - -def _granted_view_ids(session): - """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. - - ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above - takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with - the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to - answer "which views has this person starred", so the derivation moves here beside the reader - it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). - - ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming - me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to - agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test - would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so - it costs a dict lookup per id. - ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. - """ - import core.shares as shares - try: - return {str(v) for v in - ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) - .get("view") or []) - if shares.role_for("view", str(v), session.uname, - st=session.runtime) in ("view", "edit")} - except Exception: - return set() - - -@router.get("/nav") -def nav(session: Session = Depends(require_session)): - """`{pages: [{key,label,source?,chrome}], landing}`. - - Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you - are not allowed". A session that may open nothing at all is a misconfigured account, and it - gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is - how a permission bug hides in plain sight (X2's never-an-empty-200 rule). - """ - pages = list(perms.nav_pages(session.user) or []) - # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. - # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything - # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is - # its own databases, which is what "different set of databases at later waves" means. - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. - # - # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all - # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs - # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a - # row it removes and a row a store failure dropped are the SAME absence on the wire, and the - # client renders `null` for both — pixel-identical to still-loading. Naming the removals is - # what lets the shell tell "not part of this workspace" from "we could not read it". - _omitted = [] - if tmods != "all": - allowed = {str(k) for k in (tmods or [])} - _omitted = sorted({str(p.get("key")) for p in pages - if p.get("key") not in allowed - and (p.get("parent") or "") not in allowed}) - pages = [p for p in pages - if p.get("key") in allowed or (p.get("parent") or "") in allowed] - # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — - # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so - # the nav cannot offer a row the table routes would refuse. - # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. - # - # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one - # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below - # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms - # in-process** — the second figure is the one that matters, because with the network gone and - # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every - # other collapses to 20–161 ms. That is per-call CPU, and this is it. - # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ - # manage loop below consumes this same dict. - _ut_defs, _ut_locked_mode, _degraded = {}, "", [] - try: - import core.user_tables as user_tables - # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below - # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and - # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the - # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of - # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms - # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds - # apart because of CSS or ordering, they arrived when their route finished copying. - # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that - # needs a row, it raises with the reason instead of painting an empty grid. Use - # `all_tables` for that, and know you are buying the whole copy back. - _ut_defs = user_tables.all_defs(st=session.runtime) or {} - # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its - # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked - # the moment this import failed. A sentinel that can equal real data is not a sentinel. - _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) - _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=_lent): - pages.append({"key": e["key"], "label": e["label"], - "source": e.get("source") or "Blank", "chrome": "main"}) - except Exception: - # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. - # - # A store blip must not take the whole nav down with it, and that half stands. What - # changed is that it used to answer **200 OK with every database missing** and say - # nothing — so a client cannot tell "this tenant has no databases" from "we could not - # read them", and the rail renders the same complete-looking thing either way. That is - # the owner's *"Connectors and Automation module still disappears"* class of report: - # the payload is a claim about what exists, and a claim it could not verify has to be - # marked as such. `degraded` is that mark; the shell renders it in the affected slot. - _degraded.append("databases") - pass - if not pages: - if tmods != "all": - # A provisioned tenant with no modules and no databases YET is a legitimate empty - # state, not a misconfigured account — the client renders "create your first - # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is - # empty rather than leaving 200-[] ambiguous. - return {"pages": [], "landing": None, "empty": "no_databases"} - raise err(403, "no_surfaces", - "your account has no dashboards assigned. Ask an administrator.") - # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry - # rows, after the tenant module filter, after the user tables. Merged HERE rather than - # applied by the client for one reason: `label` is what every reader of this payload shows, - # including the Settings modal's `moduleLabels` map, so a client-side merge would put the - # override in one door and the registry label in the other. - # - # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A - # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If - # either ever starts handing back cached or module-level dicts, this loop would write one - # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant - # leak with no symptom until two tenants rename the same registry key. Copy the rows before - # merging on the day that invariant changes. - meta = _read_nav_meta(session.runtime) - # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. - # - # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS - # ORDER so a cold container would reach the databases somebody actually works in first. Every - # one of those was a correct answer to the wrong question: the block was `N` per-database store - # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block - # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per - # `/nav`** before this ticket, **0 and 0** after. - # - # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer - # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. - # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave - # 23), and it was only computed this early so the deleted budget could spend itself in its order. - _page_keys = {str(p.get("key", "")) for p in pages} - recents = _read_recents(session.runtime, session.uname, _page_keys) - # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. - # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. - # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, - # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. - # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an - # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. - for p in pages: - # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because - # the server is the only end that knows — the client cannot see who created a user - # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control - # only where the write would actually land, and the route re-checks it regardless. - # - # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the - # de5037f share-grant admission widened `may_open`, so a row's presence now includes - # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are - # therefore answered from the DEFINITION: creator-or-admin strictly, matching the - # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no - # controls — the honest shape (the old code offered rename to users the route 403'd). - key = str(p.get("key", "")) - if key.startswith("ut_"): - _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") - _mine = bool(session.admin or (_creator and _creator == session.uname)) - p["manage"] = _mine - p["canDelete"] = _mine - # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. - # - # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md - # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still - # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the - # live example; their rows arrive from the engine, so a "+" row there could only - # refuse, which R8 calls a fake affordance. - # - # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is - # inlined only because `_ut_defs` is already in hand — calling the predicate per row - # would be one store read per database on every nav request — and the VALUE comes - # from the module's own constant rather than a copied string, so the two cannot drift - # to different answers. If that predicate ever grows a second condition, this must - # become a call. - # - # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is - # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while - # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs - # a missing hint, never a write that should not have landed. - if (_ut_locked_mode - and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): - p["locked"] = True - elif session.admin: - p["manage"] = True - # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database - # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer - # never had to test for the key. R4 retires the count from the rail and the flyout entirely - # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. - # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an - # absent key reads as `undefined` there, which is the same thing the optional field already - # meant for a non-database row. Flagged to A rather than assumed harmless. - entry = meta.get(p.get("key")) if meta else None - if not entry: - continue - if entry.get("icon"): - p["icon"] = entry["icon"] - if entry.get("name"): - p["label"] = entry["name"] - landing = perms.landing_page(session.user) - if landing and not any(p.get("key") == landing for p in pages): - landing = pages[0].get("key") - # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks - # for. A second round trip for a list this short, computed from a bucket this route is - # already holding the store open for, would be a request per page load for no gain. - # - # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases - # included. Pruning against `perms.nav_pages` alone would silently drop every user database - # from Home's recents: the exact surface R7 is about, invisible, with every gate green. - # - # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — - # had the identical bug and nobody connected the two for four waves. It is fixed at the - # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so - # both doors finally agree on what a placeable key is. This block keeps using the assembled - # list because it already holds it: re-enumerating here would be a second store read for an - # answer sitting in a local variable. - # - # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, - # because the important-count block spends its budget in RECENTS ORDER (see there). It is still - # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here - # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. - # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. - # `omitted` — this workspace's catalogue does not include these modules. Deliberate. - # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says - # so in the affected slot instead of rendering a confident nothing. - # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as - # `[]` rather than being omitted when empty, which is the same rule `limits` follows. - return {"pages": pages, "landing": landing, "recents": recents, - "omitted": _omitted, "degraded": _degraded} - - -@router.post("/nav/opened") -def nav_opened(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. - - The client calls this on every route commit, so this is the only write in this file on a - hot path, and three things follow from that: - - · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per - page open against an HF-Dataset-backed store would put a network round trip inside every - navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; - this is. - · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string - would make the stamp cost more than the navigation that triggered it — and it would buy - nothing, because the READ prunes to what the session may currently see. An unknown or - revoked key is stored and never served back. - · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client - grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to - go, which is the same rule the read applies. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip()[:60] - if not key: - raise err(400, "bad_request", "no page was named") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was recorded.") - stamp, uname = _now(), session.uname - - def _up(data): - data = data if isinstance(data, dict) else {} - mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} - mine[key] = stamp - if len(mine) > _MAX_RECENTS: - # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape - # sorts as 0 and is the first thing evicted, which is the right answer for a value - # this route can no longer read. - def _at(item): - try: - return int(item[1]) - except (TypeError, ValueError): - return 0 - mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) - data[uname] = mine - return data - - try: - session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was recorded.") - return {"key": key, "at": stamp} - - -@router.post("/nav/meta") -def save_nav_meta(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. - - A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the - asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so - replacing the document whole is what makes a deleted folder stay deleted. This bucket is - shared by every admin in the tenant: a wholesale write here means whoever saves last - silently erases what the other one named while their tab was open. - - THREE WALLS, all fail-closed: - · the SESSION must be able to open the key at all (the same predicate the nav uses, so - the rail and this route cannot disagree about what exists); - · the WRITE is admin-only — renaming a database is a change every user in the tenant - sees, which is the definition of an administrative act here; - · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry - literals: honouring an override would leave this payload and `core/registry.py` - calling the same module two different things, and the wave-16 lesson is that the - client must not be the place that decides what a payload meant. - - `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an - icon change two independent writes rather than a race between them. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip() - if not key: - raise err(400, "bad_request", "no database was named") - # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── - # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. - # - # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what - # `user_tables.may_open` already means. A database you made is yours to name; requiring - # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` - # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, - # because a user table is not a module. Same split `/nav/schema/{key}` makes. - # · everything else — admin only, and icon only. There is no owner of `customer_data` to - # defer to, and its label is a compiled registry literal (refused below regardless). - if key.startswith("ut_"): - import core.user_tables as user_tables - if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( - key, session.uname, session.admin, st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - else: - if not session.admin: - raise err(403, "forbidden", - "only an administrator can change a built-in database's icon") - session.require(key) - - patch = {} - if "icon" in body: - icon = _clean_icon(body.get("icon")) - if body.get("icon") is not None and icon is None: - # Loud, not silent. A shape this build does not know is a CLIENT that has drifted - # from this whitelist, and answering 200 to a write that stored nothing is how - # that drift stays invisible until a user reports "my icon keeps resetting". - raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") - patch["icon"] = icon - if "name" in body: - if not key.startswith("ut_"): - raise err(400, "name_not_allowed", - "only a database you created can be renamed. This one's name comes " - "from the module registry") - name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] - if not name: - raise err(400, "bad_request", "a database needs a name") - patch["name"] = name - if not patch: - raise err(400, "bad_request", "nothing to change") - - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} - for field, value in patch.items(): - if value is None: - entry.pop(field, None) # an explicit null CLEARS - else: - entry[field] = value - # An entry with nothing left in it is removed rather than stored empty: the read side - # skips empties anyway, and a bucket that accumulates `{}` per key is a document that - # grows forever and says nothing. - if entry: - data[key] = entry - else: - data.pop(key, None) - return data - - try: - session.runtime.update(_NAV_META_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the change was not saved: the store refused the write.") - return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} - - -@router.get("/nav/prefs") -def nav_prefs(session: Session = Depends(require_session)): - """This user's database-list folders, re-validated at serve time against what they may - currently see (a revoked page's placement vanishes with the page, and returns with it).""" - try: - stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} - except Exception: - stored = {} - keys, enumerated = _placeable_top_keys(session) - return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} - - -@router.post("/nav/prefs") -def save_nav_prefs(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" - keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: - data[session.uname] = clean - else: - data.pop(session.uname, None) - return data - - try: - session.runtime.update(_NAV_PREFS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") - return {"prefs": clean} - - -@router.get("/nav/schema/{key}") -def nav_schema(key: str, session: Session = Depends(require_session)): - """The database's schema drawer payload: its field contract + the semantic measures this - session may build with. Fail-closed on the SAME predicate as the nav — a key the session - may not open answers 403, never a redacted schema.""" - # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate - # (`may_open`) rather than the module grant machinery — `session.require` would 403 every - # ut key because a user table is deliberately not a module. - if key.startswith("ut_"): - import core.user_tables as user_tables - defn = user_tables.get(key, st=session.runtime) - if not defn or not user_tables.may_open(key, session.uname, session.admin, - st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a - # schema panel opened from it that says another is the drift a rename is supposed to - # remove, not create. - return {"key": key, - "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") - or defn.get("label") or key), - "source": defn.get("source") or "Blank", - "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], - "source": f.get("source") or "overlay", - "description": str(f.get("description") or "")} - for f in (defn.get("fields") or [])], - "measures": []} - session.require(key) - pages = perms.nav_pages(session.user) or [] - page = next((p for p in pages if p.get("key") == key), None) - if page is None: - raise err(404, "unknown_page", f"{key!r} is not a database this session can see") - fields = [] - if key in ("customer_data", "cohort", "customers"): - try: - import aios_grid - for f in aios_grid.FIELDS: - entry = {"key": f["key"], "label": f["label"], "type": f["type"], - "source": f["source"], - "description": str(f.get("description") or "")} - if f.get("options"): - entry["options"] = list(f["options"]) - fields.append(entry) - except Exception: - fields = [] - measures = [] - try: - from core import measure_resolve - team_id = perms.scope_team_id(session.user) - for m in measure_resolve.offer(team_id) or []: - measures.append({"key": str(m.get("key") or ""), - "label": str(m.get("label") or m.get("key") or ""), - "type": str(m.get("type") or "")}) - except Exception: - measures = [] - out = {"key": key, "label": page.get("label") or key, - "source": page.get("source") or "", "fields": fields, "measures": measures} - if not fields: - # Honest, never a mock: a database whose contract is not yet published says so. - out["note"] = "This database has not published a field contract yet." - return out +"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. + +SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who +may see what — a nav that hides a link the API would still serve is a UI courtesy, not a +permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), +so the nav and the 403 can never disagree about a grant. + +The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no +`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the +`prefs.json` Library preference is deliberately NOT applied — is documented on +`core.perms.nav_pages`, which is where it belongs: one place, both callers. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, perms, require_session + +router = APIRouter(prefix="/api/v1") + +#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder +#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a +#: surface (the server-filtered nav still decides what exists). +_NAV_PREFS_KEY = "nav_prefs" +_MAX_NAV_FOLDERS = 16 + +#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. +#: +#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a +#: separate bucket rather than another field in that one. Folders and placement are one +#: person's arrangement of their own rail — per-user by definition. What a database is CALLED +#: and what it looks like are facts about the database: a workspace where two people call the +#: same table different things has no shared vocabulary left to discuss it in. Same store, two +#: buckets, because they answer to two different owners. +#: +#: Shape: {"": {"icon": {"shape": , "tone": }, +#: "name": ""}} +_NAV_META_KEY = "nav_meta" + +#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these +#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, +#: so it is the one place the list is written twice. +#: +#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by +#: shape and maps the result, so a shape this whitelist let through and the renderer does not +#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the +#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of +#: that. If the two lists ever drift the symptom is an icon that silently reverts to the +#: default, which is the loudest SAFE failure available here. +_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", + "chart", "map", "users", "clock", "heart", "bolt"}) +_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) +_MAX_NAV_NAME = 60 + +#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. +#: +#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is +#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature +#: rather than a convenience one. +#: +#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. +#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An +#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one +#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls +#: off the list while forty slots hold repeat visits to the first. Keying by page makes +#: "recent" mean what the word means, and makes the cap bound the number of DATABASES +#: remembered rather than the number of clicks. +_NAV_RECENTS_KEY = "nav_recents" +_MAX_RECENTS = 50 + +#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a +#: precedent, not a preference. `user_tables.create` writes +#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses +#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 +#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — +#: with nothing to go red, because both ends are internally consistent. An integer instant has +#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect +#: shipped; this is that lesson applied before it.) +def _now() -> int: + return int(time.time()) + + +def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): + """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. + + `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown + or invisible key is dropped (it can return when the grant does — placement is cosmetic, + so pruning is loss-free). Unknown folder refs drop the placement, not the folder. + + `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the + `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than + pruned: keeping a placement whose database may since have been deleted is cosmetically + harmless, while pruning a live one is the silent data loss this function just stopped + causing. Never widened to non-`ut_` keys — those come from the compiled registry, which + cannot fail to enumerate. + """ + raw = raw if isinstance(raw, dict) else {} + folders, seen = [], set() + for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:40] + name = " ".join(str(f.get("name") or "").split())[:40] + if not fid or fid in seen or not name: + continue + seen.add(fid) + folders.append({"id": fid, "name": name}) + ids = {f["id"] for f in folders} + placement = {} + src = raw.get("placement") + if isinstance(src, dict): + for k, v in src.items(): + k, v = str(k)[:60], str(v)[:40] + if v not in ids: + continue # the folder is gone; the placement goes with it + if page_keys is None or k in page_keys: + placement[k] = v + elif keep_unknown_ut and k.startswith("ut_"): + placement[k] = v + return {"folders": folders, "placement": placement} + + +def _placeable_top_keys(session): + """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. + + ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by + construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and + `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's + user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. + So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement + naming one — on READ and on WRITE. + + The symptom was "a database I drag into a folder falls out of it later", never "it does not + work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy + (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. + The next reload served the pruned copy and the database was back at root. Built-in modules + (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly + why it read as intermittent rather than as a missing feature. + + ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents + block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the + ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 + fix did not reach. + + `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set + as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement + because the store blinked would be the original defect wearing a different cause. + """ + pages = perms.nav_pages(session.user) or [] + keys = {p["key"] for p in pages if not p.get("parent")} + try: + import core.user_tables as user_tables + # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never + # name a database this session cannot see, and the nav and this door agree by sharing + # one predicate rather than by two lists being kept in step. + # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is + # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires + # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured + # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the + # owner reports as "the Automation row arrives ten seconds late". + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + keys.add(str(e.get("key") or "")) + except Exception: + return keys, False + return keys, True + + +def _clean_icon(raw): + """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" + if not isinstance(raw, dict): + return None + shape, tone = raw.get("shape"), raw.get("tone") + if shape not in _ICON_SHAPES or tone not in _ICON_TONES: + return None + return {"shape": shape, "tone": tone} + + +def _read_nav_meta(runtime): + """The tenant's `nav_meta`, validated on the way OUT as well as in. + + Re-validating a read looks redundant and is not: the bucket outlives this code, an older + build may have written a shape this one no longer accepts, and the whitelist is mirrored + from a vocabulary that lives in another language. A row whose icon does not survive + validation renders the default mark — never a crash, and never a half-drawn glyph. + """ + try: + stored = runtime.get(_NAV_META_KEY) or {} + except Exception: + return {} # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return {} + out = {} + for key, meta in stored.items(): + if not isinstance(meta, dict): + continue + entry = {} + icon = _clean_icon(meta.get("icon")) + if icon: + entry["icon"] = icon + # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write + # route refuses a name for a built-in key, but a record written by an older build (or + # by hand) could still carry one — and honouring it would let the store rename + # `customer_data` on screen while `core/registry.py` and every other reader went on + # calling it Customer. A name that could not be written today is not served today. + name = meta.get("name") + if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): + entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] + if entry: + out[str(key)] = entry + return out + + +def _read_recents(runtime, uname, allowed): + """This user's recents, newest first, PRUNED to what they may currently see. + + ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: + + · the WRITE happens on every route open — it is the one hot path this file has — so it + must not build the whole nav to validate one key; + · a key whose grant was REVOKED must stop being offered without anyone running a + migration, and must come back if the grant does. That is `_clean_nav_prefs`' own + prune-never-invent posture, applied to a second bucket for the same reason. + + A key that is not in `allowed` therefore reaches the store and never reaches a screen — + which also means a stuffed key cannot be used to discover what exists: it comes back only + if the session could already see it. + """ + try: + stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} + except Exception: + return [] # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return [] + out = [] + for key, at in stored.items(): + key = str(key) + if allowed is not None and key not in allowed: + continue + try: + at = int(at) + except (TypeError, ValueError): + continue # a stamp this build cannot read is not a stamp + if at <= 0: + continue + out.append({"key": key, "at": at}) + out.sort(key=lambda r: r["at"], reverse=True) + return out[:_MAX_RECENTS] + + +# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── +# +# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the +# block that computed them (`_important_counts`) went with them. It read one store bucket PER +# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database +# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on +# **14** pages. It is **0 and 0** now. +# +# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the +# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold +# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. +# A budget that can be exhausted is a number that can be silently short, which is why it also had to +# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to +# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge +# instead of a late rail. Closes D-288 and D-289. +# +# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the +# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store +# read was — and duplicating them into the new route would have put two answers behind one badge. + +#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the +#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> +#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is +#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second +#: copy here would be free to drift from it ([[one-question-two-normalizers]]). +_WS_SUFFIX = "_table_workspace" + + +def _visible_views(doc, uname, is_admin, granted_ids): + """Every view on ONE database that THIS caller can see, from an already-read workspace doc. + + ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get + wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` + — one home per view, never both — so "how many views are marked important here" has a + DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and + `admin` has none, on the same database. A tenant-wide count would put a number in the owner's + rail that no view sidebar they can open would ever add up to. + + ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for + it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole + bucket per database (the N whole reads this block exists to avoid), and the only other option is + a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told + exists; a private import cannot. + """ + import core.table_store as table_store + out = {} + for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): + if isinstance(view, dict): + out[str(vid)] = view + for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): + if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): + out.setdefault(str(vid), view) + # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the + # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. + if granted_ids: + for stratum, blob in doc.items(): + if stratum == uname or not isinstance(blob, dict): + continue + for vid, view in (blob.get("views") or {}).items(): + if str(vid) in granted_ids and isinstance(view, dict): + out.setdefault(str(vid), view) + return out + + +def _view_record_count(cfg, cohorts): + """How many RECORDS this view resolves to, or None when that cannot be answered for free. + + ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact + because the view CARRIES its row set; the third — an ordinary filtered view — can only be + counted by running its filters over the records, and the records are the one thing this route + must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` + here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and + the database's `partial` flag says so, which is the whole of R6's second sentence applied to a + badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a + number that is short by an unknown amount. + + ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` + counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); + this end is the exact inverse — it has no rows at all and no window either. The two are honest + about different halves, which is why `partial` had to be on the wire rather than derived. + """ + if not isinstance(cfg, dict): + return None + # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` + # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + n = cohorts.get(lock) + return int(n) if isinstance(n, int) else None + # A curated row set carries its own count. + pids = cfg.get("memberPids") + if isinstance(pids, list) and pids: + return len(pids) + return None + + +def _visible_database_keys(session): + """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. + + ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: + + · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT + CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. + A star scan cannot borrow that: it would open the view bucket of a database this workspace's + catalogue does not include. `nav()` applies the catalogue filter; that helper never has. + · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so + it cannot be reached from another route without rebuilding the payload. + + So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same + order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the + tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes + the answer honest for a caller about to read a view bucket: a module surface with no workspace + (`sales`, `ar`) is not a database and has no views to star. + + ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` + excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: + a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even + though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious + reading — this ticket's first draft assumed the clause was excluding children and wrote a + docstring around a defect that does not exist. + + ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as + authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. + """ + import core.view_templates as view_templates + keys, enumerated = set(), True + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} + for p in (perms.nav_pages(session.user) or []): + key = str(p.get("key") or "") + if allowed is not None and key not in allowed \ + and str(p.get("parent") or "") not in allowed: + continue + if view_templates.workspace_key(key): + keys.add(key) + try: + import core.user_tables as user_tables + # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection + # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + k = str(e.get("key") or "") + if k: + keys.add(k) + except Exception: + enumerated = False + return keys, enumerated + + +def _granted_view_ids(session): + """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. + + ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above + takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with + the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to + answer "which views has this person starred", so the derivation moves here beside the reader + it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). + + ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming + me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to + agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test + would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so + it costs a dict lookup per id. + ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. + """ + import core.shares as shares + try: + return {str(v) for v in + ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) + .get("view") or []) + if shares.role_for("view", str(v), session.uname, + st=session.runtime) in ("view", "edit")} + except Exception: + return set() + + +@router.get("/nav") +def nav(session: Session = Depends(require_session)): + """`{pages: [{key,label,source?,chrome}], landing}`. + + Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you + are not allowed". A session that may open nothing at all is a misconfigured account, and it + gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is + how a permission bug hides in plain sight (X2's never-an-empty-200 rule). + """ + pages = list(perms.nav_pages(session.user) or []) + # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. + # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything + # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is + # its own databases, which is what "different set of databases at later waves" means. + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. + # + # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all + # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs + # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a + # row it removes and a row a store failure dropped are the SAME absence on the wire, and the + # client renders `null` for both — pixel-identical to still-loading. Naming the removals is + # what lets the shell tell "not part of this workspace" from "we could not read it". + _omitted = [] + if tmods != "all": + allowed = {str(k) for k in (tmods or [])} + _omitted = sorted({str(p.get("key")) for p in pages + if p.get("key") not in allowed + and (p.get("parent") or "") not in allowed}) + pages = [p for p in pages + if p.get("key") in allowed or (p.get("parent") or "") in allowed] + # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — + # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so + # the nav cannot offer a row the table routes would refuse. + # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. + # + # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one + # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below + # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms + # in-process** — the second figure is the one that matters, because with the network gone and + # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every + # other collapses to 20–161 ms. That is per-call CPU, and this is it. + # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ + # manage loop below consumes this same dict. + _ut_defs, _ut_locked_mode, _degraded = {}, "", [] + try: + import core.user_tables as user_tables + # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below + # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and + # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the + # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of + # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms + # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds + # apart because of CSS or ordering, they arrived when their route finished copying. + # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that + # needs a row, it raises with the reason instead of painting an empty grid. Use + # `all_tables` for that, and know you are buying the whole copy back. + _ut_defs = user_tables.all_defs(st=session.runtime) or {} + # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its + # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked + # the moment this import failed. A sentinel that can equal real data is not a sentinel. + _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) + _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=_lent): + pages.append({"key": e["key"], "label": e["label"], + "source": e.get("source") or "Blank", "chrome": "main"}) + except Exception: + # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. + # + # A store blip must not take the whole nav down with it, and that half stands. What + # changed is that it used to answer **200 OK with every database missing** and say + # nothing — so a client cannot tell "this tenant has no databases" from "we could not + # read them", and the rail renders the same complete-looking thing either way. That is + # the owner's *"Connectors and Automation module still disappears"* class of report: + # the payload is a claim about what exists, and a claim it could not verify has to be + # marked as such. `degraded` is that mark; the shell renders it in the affected slot. + _degraded.append("databases") + pass + if not pages: + if tmods != "all": + # A provisioned tenant with no modules and no databases YET is a legitimate empty + # state, not a misconfigured account — the client renders "create your first + # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is + # empty rather than leaving 200-[] ambiguous. + return {"pages": [], "landing": None, "empty": "no_databases"} + raise err(403, "no_surfaces", + "your account has no dashboards assigned. Ask an administrator.") + # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry + # rows, after the tenant module filter, after the user tables. Merged HERE rather than + # applied by the client for one reason: `label` is what every reader of this payload shows, + # including the Settings modal's `moduleLabels` map, so a client-side merge would put the + # override in one door and the registry label in the other. + # + # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A + # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If + # either ever starts handing back cached or module-level dicts, this loop would write one + # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant + # leak with no symptom until two tenants rename the same registry key. Copy the rows before + # merging on the day that invariant changes. + meta = _read_nav_meta(session.runtime) + # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. + # + # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS + # ORDER so a cold container would reach the databases somebody actually works in first. Every + # one of those was a correct answer to the wrong question: the block was `N` per-database store + # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block + # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per + # `/nav`** before this ticket, **0 and 0** after. + # + # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer + # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. + # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave + # 23), and it was only computed this early so the deleted budget could spend itself in its order. + _page_keys = {str(p.get("key", "")) for p in pages} + recents = _read_recents(session.runtime, session.uname, _page_keys) + # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. + # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. + # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, + # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. + # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an + # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. + for p in pages: + # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because + # the server is the only end that knows — the client cannot see who created a user + # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control + # only where the write would actually land, and the route re-checks it regardless. + # + # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the + # de5037f share-grant admission widened `may_open`, so a row's presence now includes + # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are + # therefore answered from the DEFINITION: creator-or-admin strictly, matching the + # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no + # controls — the honest shape (the old code offered rename to users the route 403'd). + key = str(p.get("key", "")) + if key.startswith("ut_"): + _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") + _mine = bool(session.admin or (_creator and _creator == session.uname)) + p["manage"] = _mine + p["canDelete"] = _mine + # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. + # + # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md + # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still + # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the + # live example; their rows arrive from the engine, so a "+" row there could only + # refuse, which R8 calls a fake affordance. + # + # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is + # inlined only because `_ut_defs` is already in hand — calling the predicate per row + # would be one store read per database on every nav request — and the VALUE comes + # from the module's own constant rather than a copied string, so the two cannot drift + # to different answers. If that predicate ever grows a second condition, this must + # become a call. + # + # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is + # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while + # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs + # a missing hint, never a write that should not have landed. + if (_ut_locked_mode + and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): + p["locked"] = True + elif session.admin: + p["manage"] = True + # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database + # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer + # never had to test for the key. R4 retires the count from the rail and the flyout entirely + # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. + # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an + # absent key reads as `undefined` there, which is the same thing the optional field already + # meant for a non-database row. Flagged to A rather than assumed harmless. + entry = meta.get(p.get("key")) if meta else None + if not entry: + continue + if entry.get("icon"): + p["icon"] = entry["icon"] + if entry.get("name"): + p["label"] = entry["name"] + landing = perms.landing_page(session.user) + if landing and not any(p.get("key") == landing for p in pages): + landing = pages[0].get("key") + # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks + # for. A second round trip for a list this short, computed from a bucket this route is + # already holding the store open for, would be a request per page load for no gain. + # + # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases + # included. Pruning against `perms.nav_pages` alone would silently drop every user database + # from Home's recents: the exact surface R7 is about, invisible, with every gate green. + # + # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — + # had the identical bug and nobody connected the two for four waves. It is fixed at the + # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so + # both doors finally agree on what a placeable key is. This block keeps using the assembled + # list because it already holds it: re-enumerating here would be a second store read for an + # answer sitting in a local variable. + # + # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, + # because the important-count block spends its budget in RECENTS ORDER (see there). It is still + # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here + # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. + # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. + # `omitted` — this workspace's catalogue does not include these modules. Deliberate. + # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says + # so in the affected slot instead of rendering a confident nothing. + # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as + # `[]` rather than being omitted when empty, which is the same rule `limits` follows. + return {"pages": pages, "landing": landing, "recents": recents, + "omitted": _omitted, "degraded": _degraded} + + +@router.post("/nav/opened") +def nav_opened(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. + + The client calls this on every route commit, so this is the only write in this file on a + hot path, and three things follow from that: + + · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per + page open against an HF-Dataset-backed store would put a network round trip inside every + navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; + this is. + · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string + would make the stamp cost more than the navigation that triggered it — and it would buy + nothing, because the READ prunes to what the session may currently see. An unknown or + revoked key is stored and never served back. + · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client + grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to + go, which is the same rule the read applies. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip()[:60] + if not key: + raise err(400, "bad_request", "no page was named") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was recorded.") + stamp, uname = _now(), session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + mine[key] = stamp + if len(mine) > _MAX_RECENTS: + # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape + # sorts as 0 and is the first thing evicted, which is the right answer for a value + # this route can no longer read. + def _at(item): + try: + return int(item[1]) + except (TypeError, ValueError): + return 0 + mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) + data[uname] = mine + return data + + try: + session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was recorded.") + return {"key": key, "at": stamp} + + +@router.post("/nav/meta") +def save_nav_meta(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. + + A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the + asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so + replacing the document whole is what makes a deleted folder stay deleted. This bucket is + shared by every admin in the tenant: a wholesale write here means whoever saves last + silently erases what the other one named while their tab was open. + + THREE WALLS, all fail-closed: + · the SESSION must be able to open the key at all (the same predicate the nav uses, so + the rail and this route cannot disagree about what exists); + · the WRITE is admin-only — renaming a database is a change every user in the tenant + sees, which is the definition of an administrative act here; + · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry + literals: honouring an override would leave this payload and `core/registry.py` + calling the same module two different things, and the wave-16 lesson is that the + client must not be the place that decides what a payload meant. + + `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an + icon change two independent writes rather than a race between them. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip() + if not key: + raise err(400, "bad_request", "no database was named") + # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── + # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. + # + # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what + # `user_tables.may_open` already means. A database you made is yours to name; requiring + # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` + # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, + # because a user table is not a module. Same split `/nav/schema/{key}` makes. + # · everything else — admin only, and icon only. There is no owner of `customer_data` to + # defer to, and its label is a compiled registry literal (refused below regardless). + if key.startswith("ut_"): + import core.user_tables as user_tables + if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( + key, session.uname, session.admin, st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + else: + if not session.admin: + raise err(403, "forbidden", + "only an administrator can change a built-in database's icon") + session.require(key) + + patch = {} + if "icon" in body: + icon = _clean_icon(body.get("icon")) + if body.get("icon") is not None and icon is None: + # Loud, not silent. A shape this build does not know is a CLIENT that has drifted + # from this whitelist, and answering 200 to a write that stored nothing is how + # that drift stays invisible until a user reports "my icon keeps resetting". + raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") + patch["icon"] = icon + if "name" in body: + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database you created can be renamed. This one's name comes " + "from the module registry") + name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] + if not name: + raise err(400, "bad_request", "a database needs a name") + patch["name"] = name + if not patch: + raise err(400, "bad_request", "nothing to change") + + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + for field, value in patch.items(): + if value is None: + entry.pop(field, None) # an explicit null CLEARS + else: + entry[field] = value + # An entry with nothing left in it is removed rather than stored empty: the read side + # skips empties anyway, and a bucket that accumulates `{}` per key is a document that + # grows forever and says nothing. + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + session.runtime.update(_NAV_META_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} + + +@router.get("/nav/prefs") +def nav_prefs(session: Session = Depends(require_session)): + """This user's database-list folders, re-validated at serve time against what they may + currently see (a revoked page's placement vanishes with the page, and returns with it).""" + try: + stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} + except Exception: + stored = {} + keys, enumerated = _placeable_top_keys(session) + return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} + + +@router.post("/nav/prefs") +def save_nav_prefs(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Wholesale replace, like the table folder stratum — the client sent its complete + picture, validated here; a partial merge would resurrect deleted folders forever.""" + keys, enumerated = _placeable_top_keys(session) + clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + if clean["folders"] or clean["placement"]: + data[session.uname] = clean + else: + data.pop(session.uname, None) + return data + + try: + session.runtime.update(_NAV_PREFS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the folder change was not saved: the store refused the write.") + return {"prefs": clean} + + +@router.get("/nav/schema/{key}") +def nav_schema(key: str, session: Session = Depends(require_session)): + """The database's schema drawer payload: its field contract + the semantic measures this + session may build with. Fail-closed on the SAME predicate as the nav — a key the session + may not open answers 403, never a redacted schema.""" + # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate + # (`may_open`) rather than the module grant machinery — `session.require` would 403 every + # ut key because a user table is deliberately not a module. + if key.startswith("ut_"): + import core.user_tables as user_tables + defn = user_tables.get(key, st=session.runtime) + if not defn or not user_tables.may_open(key, session.uname, session.admin, + st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a + # schema panel opened from it that says another is the drift a rename is supposed to + # remove, not create. + return {"key": key, + "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") + or defn.get("label") or key), + "source": defn.get("source") or "Blank", + "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], + "source": f.get("source") or "overlay", + "description": str(f.get("description") or "")} + for f in (defn.get("fields") or [])], + "measures": []} + session.require(key) + pages = perms.nav_pages(session.user) or [] + page = next((p for p in pages if p.get("key") == key), None) + if page is None: + raise err(404, "unknown_page", f"{key!r} is not a database this session can see") + fields = [] + if key in ("customer_data", "cohort", "customers"): + try: + import aios_grid + for f in aios_grid.FIELDS: + entry = {"key": f["key"], "label": f["label"], "type": f["type"], + "source": f["source"], + "description": str(f.get("description") or "")} + if f.get("options"): + entry["options"] = list(f["options"]) + fields.append(entry) + except Exception: + fields = [] + measures = [] + try: + from core import measure_resolve + team_id = perms.scope_team_id(session.user) + for m in measure_resolve.offer(team_id) or []: + measures.append({"key": str(m.get("key") or ""), + "label": str(m.get("label") or m.get("key") or ""), + "type": str(m.get("type") or "")}) + except Exception: + measures = [] + out = {"key": key, "label": page.get("label") or key, + "source": page.get("source") or "", "fields": fields, "measures": measures} + if not fields: + # Honest, never a mock: a database whose contract is not yet published says so. + out["note"] = "This database has not published a field contract yet." + return out diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 7795caf496a8598b3c52b4cf5b0663939012d5c8..8e49c41134aafde8a9738e53d63775c2d76fea88 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -1,114 +1,114 @@ -"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). - -Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, -shapes and status codes here; every decision that could be wrong lives in the module a gate -can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, -so the day a second provider lands here is the day nothing in this file changes. - -MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session -`main.py`, and `routes_automation` is already included there — so this router rides inside it -(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that -alters no path. - -⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent -screen** — it is a top-level navigation the client reaches by ``, never JSON. The -callback 302s BACK to the return path the `state` carried (relative-only, sanitised by -`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever -went wrong rides in the query string; a dead-end error page where the app used to be reads as -"the product broke", not "the connect failed". -""" -import os - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import RedirectResponse - -import oauth_connect -from deps import Session, err, require_session - -router = APIRouter(prefix="/oauth") - - -def _redirect_uri(request: Request, provider: str) -> str: - """The redirect URI this deployment registers at the provider — env-pinned when the - container sits behind a proxy that rewrites the scheme (the HF Space), else derived from - the request. MUST match a console-registered URI verbatim, so it is computed in exactly - one place. - - ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to - the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and - the request-derived fallback below is effectively dev-only. - ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host - this returns is where the provider sends the user BACK, and the session cookie is host-only - (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host - the user actually browsed plants the session on the wrong hostname and they return logged - out. Moving the app to a new hostname means moving this value AND re-registering the - resulting URI in the provider console; one without the other fails closed. - Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" - base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") - if not base: - base = f"{request.url.scheme}://{request.url.netloc}" - return f"{base}/api/v1/oauth/{provider}/callback" - - -@router.get("/status") -def oauth_status(session: Session = Depends(require_session)): - """C5's status shape for the SESSION user, one entry per registry provider: - `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's - `ready` reads through.""" - return oauth_connect.status(session.runtime, session.uname) - - -def _offered_or_404(provider: str): - """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. - - The owner pasted the failure this replaces: clicking Connect on the Google card answered - `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is - not offered at all, so the honest status is the one for a URL that does not exist. `404` - rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not - coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). - - ⚠ Every door in this router goes through it, START included, because the JSON the owner saw - came from the start route and a guard on one leg is a guard on one leg. - """ - if not oauth_connect.offered(provider): - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - - -@router.get("/{provider}/start") -def oauth_start(provider: str, request: Request, next: str = "", - session: Session = Depends(require_session)): - """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the - RELATIVE path the callback returns the browser to; it rides inside the single-use state, - sanitised, so the round trip cannot be steered off-origin.""" - _offered_or_404(provider) - url, problem = oauth_connect.start(provider, session.uname, - _redirect_uri(request, provider), next_path=next) - if problem: - raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) - return RedirectResponse(url, status_code=302) - - -@router.get("/{provider}/callback") -def oauth_callback(provider: str, request: Request, - session: Session = Depends(require_session), - state: str = "", code: str = "", error: str = ""): - """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends - the browser back to the state's return path — connected or not (see module header).""" - _offered_or_404(provider) - if error: - home = "/#/" - return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) - email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) - sep = "&" if "?" in home else "?" - if problem: - return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) - return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) - - -@router.post("/{provider}/disconnect") -def oauth_disconnect(provider: str, session: Session = Depends(require_session)): - _offered_or_404(provider) - if oauth_connect.provider_def(provider) is None: - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - oauth_connect.disconnect(session.runtime, session.uname, provider) - return {"disconnected": provider} +"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). + +Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, +shapes and status codes here; every decision that could be wrong lives in the module a gate +can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, +so the day a second provider lands here is the day nothing in this file changes. + +MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session +`main.py`, and `routes_automation` is already included there — so this router rides inside it +(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that +alters no path. + +⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent +screen** — it is a top-level navigation the client reaches by ``, never JSON. The +callback 302s BACK to the return path the `state` carried (relative-only, sanitised by +`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever +went wrong rides in the query string; a dead-end error page where the app used to be reads as +"the product broke", not "the connect failed". +""" +import os + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import RedirectResponse + +import oauth_connect +from deps import Session, err, require_session + +router = APIRouter(prefix="/oauth") + + +def _redirect_uri(request: Request, provider: str) -> str: + """The redirect URI this deployment registers at the provider — env-pinned when the + container sits behind a proxy that rewrites the scheme (the HF Space), else derived from + the request. MUST match a console-registered URI verbatim, so it is computed in exactly + one place. + + ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to + the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and + the request-derived fallback below is effectively dev-only. + ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host + this returns is where the provider sends the user BACK, and the session cookie is host-only + (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host + the user actually browsed plants the session on the wrong hostname and they return logged + out. Moving the app to a new hostname means moving this value AND re-registering the + resulting URI in the provider console; one without the other fails closed. + Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" + base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") + if not base: + base = f"{request.url.scheme}://{request.url.netloc}" + return f"{base}/api/v1/oauth/{provider}/callback" + + +@router.get("/status") +def oauth_status(session: Session = Depends(require_session)): + """C5's status shape for the SESSION user, one entry per registry provider: + `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's + `ready` reads through.""" + return oauth_connect.status(session.runtime, session.uname) + + +def _offered_or_404(provider: str): + """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. + + The owner pasted the failure this replaces: clicking Connect on the Google card answered + `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is + not offered at all, so the honest status is the one for a URL that does not exist. `404` + rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not + coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). + + ⚠ Every door in this router goes through it, START included, because the JSON the owner saw + came from the start route and a guard on one leg is a guard on one leg. + """ + if not oauth_connect.offered(provider): + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + + +@router.get("/{provider}/start") +def oauth_start(provider: str, request: Request, next: str = "", + session: Session = Depends(require_session)): + """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the + RELATIVE path the callback returns the browser to; it rides inside the single-use state, + sanitised, so the round trip cannot be steered off-origin.""" + _offered_or_404(provider) + url, problem = oauth_connect.start(provider, session.uname, + _redirect_uri(request, provider), next_path=next) + if problem: + raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) + return RedirectResponse(url, status_code=302) + + +@router.get("/{provider}/callback") +def oauth_callback(provider: str, request: Request, + session: Session = Depends(require_session), + state: str = "", code: str = "", error: str = ""): + """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends + the browser back to the state's return path — connected or not (see module header).""" + _offered_or_404(provider) + if error: + home = "/#/" + return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) + email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) + sep = "&" if "?" in home else "?" + if problem: + return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) + return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) + + +@router.post("/{provider}/disconnect") +def oauth_disconnect(provider: str, session: Session = Depends(require_session)): + _offered_or_404(provider) + if oauth_connect.provider_def(provider) is None: + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + oauth_connect.disconnect(session.runtime, session.uname, provider) + return {"disconnected": provider} diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index bc613d0fa1f00b318cf522e13160df7e9f110881..c1d1a3b2f2d4d0d97c40f214d82ff08f2acf8ad6 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -1,1221 +1,1336 @@ -"""routes_odoo_tables.py — the door to the Odoo relational spawn (wave 27 item 17, contract C8). - -Two endpoints and no cleverness: refresh the locked Odoo databases, and report their freshness. -The work itself lives in `odoo_relational.py`; this file only decides WHO may ask and turns a -refusal into a status code. - -⛔ THE MOUNT IS DONE (`main.py` includes this router) and the wave-23 scar it was written against -— three finished routers shipping 404-dead behind green gates — is covered by `verify_api` -enumerating `main.app.routes`. ⚠ BUT WAVE 28 PROVED THAT CONTROL IS ONLY HALF OF THE QUESTION: -being mounted is not being CALLABLE. This router was mounted, enumerated, green, and answered a -plain-text 500 to every request for a day because of an attribute typo in the admin check -(`session.is_admin`, which does not exist). A route-existence check cannot see that; only calling -it can. `verify_api` now does both. - -ADMIN-GATED, and not for tidiness: a refresh REWRITES four locked databases for the whole tenant -and deletes the rows that left the population. That is an operator action. -⭐ WAVE 30 (R6/R7, contract C2) ADDS A THIRD ENDPOINT AND IT IS A DIFFERENT KIND OF THING: the -READ-THROUGH WINDOW. The two above operate on the materialised copy; `/{table_key}/rows` does not -read the copy at all. See the block above `GRID_SOURCES` for why that had to change. -""" -import json - -from fastapi import Depends -from fastapi import APIRouter, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - - -def _rel(): - import odoo_relational - return odoo_relational - - -def _rt(): - from harness import runtime - return runtime - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W31-T45 / D-169 — THE CROSS-TENANT MIRROR GUARD, AND WHY EVERY DOOR BELOW GOES THROUGH IT. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# `harness/datastore` holds ONE process-wide DuckDB connection over a process-global `DB_PATH`, -# and ONE Space process serves EVERY tenant. `TenantRuntime.assert_datastore_matches` was written -# for exactly the failure that arrangement produces — its own docstring names it: *"an analytical -# read served from another tenant's file returns real, plausible, wrong rows … and the number -# reconciles against the wrong book"* — and until this wave its only caller anywhere was -# `verify_api.py`. The guard was correct, tested, and governed nothing. -# -# ⛔ WHAT ACTUALLY HELD THE BOUNDARY WAS THE CUSTOMER LIST, NOT THE CODE. Three of the four doors -# below are reachable only by a tenant whose own `user_tables` document declares the table -# (`_defn_or_refuse`) or whose slug passes `is_royal` — so no second tenant could reach the mirror -# because no second tenant had mirror databases. R2 puts Meta Ads on this same mirror for GTM Lab, -# which ends that fact. A boundary that holds because of who our customers are is not a boundary. -# -# ⚠ AND ONE DOOR WAS ALREADY THROUGH IT TODAY — measured, not theorised: `/odoo-tables/status` -# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable` -# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row -# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes. -def _mirror_cur(session): - """A mirror cursor for THIS session's tenant, or a refusal that says which failure it is. - - Returns `(cursor, mismatch_sentence)` — exactly one of the two is None, so a caller cannot - accidentally treat "refused" as "empty". - - ⛔ IT CATCHES ONLY THE MISMATCH AND LETS THE OTHER `RuntimeError` THROUGH, because the two mean - opposite actions and every caller already has a policy for the second one: - - * `DatastoreMismatch` — this process has ANOTHER tenant's file open. It never fixes itself by - waiting; the deployment is mis-pinned. Returned as a SENTENCE so each door can choose its - own status (the row doors raise, the status door reports — R6's second sentence). - * a plain `RuntimeError` — the mirror is completing its first sync, and retrying works. It - PROPAGATES, so `odoo_table_rows` keeps answering `503 store_not_ready` and the status door - keeps degrading to "no size half", exactly as each did before this guard existed. - - ⚠ Collapsing them was the tempting simplification and it is the defect: `DatastoreMismatch` - SUBCLASSES `RuntimeError`, so a single `except RuntimeError` here would render every - cross-tenant refusal as a retry banner. Catching the subclass first is a correctness rule. - """ - rtm = _rt() - try: - return rtm.mirror_cursor(session.runtime), None - except rtm.DatastoreMismatch as e: - return None, str(e) - - -@router.post("/odoo-tables/refresh") -def refresh_odoo_tables(session: Session = Depends(require_session)): - """Rebuild all four locked Odoo databases from the tenant's mirror — customers, products, - invoices and orders (it was invoices + customers until 2026-08-09). - - Idempotent: row ids are Odoo ids, so a re-run updates in place. Returns what MOVED, because - "refreshed" with no counts is indistinguishable from a no-op over an empty mirror. - """ - # ⛔ `deps.err()` RETURNS an HTTPException, it does not raise one — so it must be `raise - # err(...)`. `return err(...)` serialises the exception object with a **200**, which is the - # wave-23 shape exactly: a finished route, green everything, wrong on the wire. The house - # convention is 243 `raise err` against 6 strays; this file uses `raise`. - rel = _rel() - # ⛔⛔ THIS LINE WAS `session.is_admin` AND IT IS WHY D-107 LOOKED LIKE A STORE PROBLEM FOR A - # DAY. `Session` is a plain dataclass with ONE admin accessor, the `.admin` property; there is - # no `is_admin` and no `__getattr__`, so the attribute lookup raised `AttributeError` — SEVEN - # LINES ABOVE the `try:` below, for every caller, admin or not. That is the whole explanation - # for the measured signature: a bare `500` carrying Starlette's default PLAIN-TEXT body - # instead of our JSON envelope, on a route whose own handler had just been taught to name the - # exception. The refresh never ran, never reached `plan()`, never touched the mirror. - # ⚠ The tell was in the diagnosis all along and was read as evidence about the STORE: "it is - # not a timeout, the same operation takes 16.8 s from a laptop". Correct, and the reason was - # that the request never got as far as doing any work at all. - # ⭐ It was the SOLE `session.is_admin` in the repo against ~40 `session.admin` — an - # unmounted-shaped defect that no gate could see, because `verify_api` pinned these two routes - # as MOUNTED and never CALLED them. That gate now calls them; see `verify_api`'s odoo-tables - # section and its negative control. - if not session.admin: - raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.") - if not rel.is_royal(session.tenant): - # A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty - # locked databases it can never fill would be worse than refusing. - raise err(400, "not_applicable", - "These databases are built from the Royal Imports Odoo mirror (R1).") - try: - out = rel.refresh(session.runtime, session.tenant, username=session.user) - # ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN. - # - # The spawn writes every bucket's rows straight into the tenant document by mutating it - # inside its own updater, so no guard in `core.user_tables` is on that path (measured, not - # assumed: `_ensure_table_inplace` never calls a function there). Stripping AFTER the write - # is what makes "a read-through grid stores no rows" true rather than intended — and the - # refresh is the only moment a stripped table can come back. - # ⚠ Reported in the response, because a silent 7 MB moving in or out of a tenant's - # document is exactly the kind of thing an operator should be able to see happening. - sync_read_through() - moved = _ut().strip_materialised(st=session.runtime) - return {"ok": True, **out, "unmaterialised": moved} - except rel.Refused as e: - # A refusal is the ANSWER, not a crash: the caller must see WHY nothing was written. - raise err(409, "refused", str(e)) - except RuntimeError as e: - # `datastore.ro_con()` raises this while the store is completing its first sync. Relaying - # it beats writing a partial table from a half-synced mirror. - raise err(503, "store_not_ready", str(e)) - except Exception as e: # noqa: BLE001 - # ⛔ AN UNEXPECTED FAILURE MUST STILL SAY WHAT IT WAS. Measured live 2026-08-09: this - # route answered a bare `500` and the operator had no way to learn why — the reason lived - # only in a container log nobody can reach from the product. A spawn that rewrites four - # locked databases is exactly the operation whose failure needs a sentence. - # ⚠ The TYPE is included deliberately: `BinderException: Referenced column "agent_id" not - # found` is a different action (wait for the column backfill) from a timeout or an auth - # failure, and "500" cannot tell them apart. - raise err(500, "refresh_failed", f"{type(e).__name__}: {e}") - - -@router.get("/odoo-tables/status") -def odoo_tables_status(session: Session = Depends(require_session)): - """Row counts + the newest `refreshed` stamp per table. - - ⚠ The stamp is the whole point while the resync wiring is outstanding: a locked collections - worklist that quietly stopped updating is a worklist that lies, so its age must be readable - without anybody running a refresh to find out. - """ - rel, out = _rel(), {} - import core.user_tables as user_tables - # ⭐ W30-T31 — THE ELIGIBILITY PASS RUNS HERE TOO, and the mirror cursor is taken best-effort: - # the freshness surface must answer on a box with no mirror (that is what it is FOR), so a - # store that is not ready costs the size half of the question, never the whole endpoint. - # ⭐⭐ W31-T45 / D-169 — THIS IS THE DOOR THAT WAS ALREADY LEAKING, and it leaked as COUNTS - # rather than as rows, which is why nothing caught it. There is no tenant gate above this - # line: `rel.is_royal(session.tenant)` appears once, as an `applicable` FIELD in the response - # 60 lines below. So a nurilab or gtmlab session took a cursor on tenant #0's `royal.duckdb`, - # `_source_for` read its columns and `_ds.window` counted its rows — real, plausible numbers - # belonging to another customer, returned 200 OK. - # ⛔ IT REPORTS RATHER THAN RAISING, deliberately, and the distinction is R6's second sentence: - # "which Odoo databases exist for me" is a legitimate question for any tenant and its honest - # answer here is *none* — a 409 would be refusing the question instead of the leak. So the - # mirror half is refused, `rowsFrom` says so per table, and the refusal rides the payload with - # its cause. A count that silently became `-1` would be the silent truncation R6 bans. - cur, mirror_refused = None, None - try: - cur, mirror_refused = _mirror_cur(session) - except Exception: # noqa: BLE001 - pass # mid-first-sync (or no mirror at all) — the document half answers - eligible, why_not = sync_read_through(cur) - # ⚠ ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)` - # while those were the only two; the 2026-08-09 widening added products and orders, and a - # hard-coded list here would have reported "everything is fine" over two databases it had - # stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count). - for _bucket, key, _label, _fields in rel.TABLES: - table = user_tables.get(key, st=session.runtime) or {} - rows = (table.get("rows") or {}) - stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()] - # ⛔⛔ A FRESHNESS SURFACE THAT LIES IS WORSE THAN NO FRESHNESS SURFACE, and this endpoint - # was one step from becoming one. Both numbers below are derived from `table["rows"]`, so - # the moment a grid stops materialising they would read `rows: 0, refreshed: ""` — an - # operator would see a database that looks EMPTY and STALE on a route whose own docstring - # says a worklist that quietly stopped updating must be legible without a refresh. So a - # read-through table is counted from the MIRROR and says where its count came from. - materialised = user_tables.materialises(key, st=session.runtime) - if not materialised: - try: - rows = {} - stamps = [] - spec = _source_for(cur, key) if cur is not None else None - if spec is not None: - from harness import datastore as _ds - frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") - else {"table": spec["table"]}) - n = _ds.window(select="1", where=spec.get("where") or "", - order_by=spec["id"], limit=1, cur=cur, **frm)["total"] - else: - n = -1 # unknown, and never reported as zero - except Exception: # noqa: BLE001 - n = -1 - out[key] = { - "exists": bool(table), - "rows": (len(rows) if materialised else n), - # ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from - # nowhere must not be labelled with the source it did not come from. - "rowsFrom": ("document" if materialised - else ("refused" if mirror_refused else "mirror")), - "materialised": materialised, - # why this grid still keeps its rows in the tenant document, when it does not have to - "materialisedBecause": (why_not.get(key, "") if materialised else ""), - "refreshed": (max([s for s in stamps if s], default="") if materialised - else "live — read through the mirror"), - "locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE, - # W30/R7: whether this database is served THROUGH the mirror rather than from the copy - # above. Reported per table, because they convert one at a time and an operator - # reading `rows: 0` needs to know whether that means "empty" or "not stored here". - "readThrough": key in _sources(), - } - # ⭐ W30-T32 / R6's SECOND SENTENCE, APPLIED TO OUR OWN HALF-BUILT STATE. A read-through - # binding whose FIELD DECLARATION has not landed yet answers 404 on the rows route, and an - # operator would read that as "the grid does not exist" rather than as "half of it shipped". - # These are the two line grains: bound here, declared in `odoo_relational` by W30-T35. - # ⚠ It reports the KEYS, never a field list — inventing a contract here is exactly the second - # source of truth the ticket forbids. - pending = {k: {"readThrough": True, "declared": False, - "cause": "this connected grid has a read-through binding but no field " - "declaration in odoo_relational yet, so it cannot be opened", - "recommendation": "declare its fields + a TABLES row (W30-T35); the binding " - "and the window are already live"} - for k in _sources() if k not in out} - return {"ok": True, "applicable": rel.is_royal(session.tenant), "tables": out, - "bound_not_declared": pending, - # ⭐ W31-T45 / D-169 — R6's SECOND SENTENCE, ON A GUARD RATHER THAN ON A ROW CAP. A - # limit that genuinely cannot be removed must be REPORTED with its cause and a - # recommended fix; silence is the violation. `null` when the mirror answered. - "mirrorRefused": ( - {"cause": mirror_refused, - "effect": "sizes_unavailable", - "recommendation": "this deployment has another tenant's analytical store open; " - "sizes are withheld rather than read from it. Pin the process " - "to this tenant (AIOS_DUCKDB_PATH / datastore.use_path) to " - "restore them."} - if mirror_refused else None), - # ⭐ W31-T46 / D-160 — WHY THERE IS NO MIRROR, when there is none. The last time this - # was silent it read as a pinned tag and a store problem for days. `null` when the - # container has one; a sentence with a cause and a fix when it does not. - "mirrorSeed": _seed_state()} - - -def _seed_state(): - """`main.MIRROR_SEED` as a reportable block, or None when the mirror is present. - - ⚠ IMPORTED LAZILY AND FAIL-QUIET: `main` imports this router, so a module-level import here - would be a cycle, and a freshness surface must never 500 because a diagnostic was unavailable. - """ - try: - import main as _main - from harness import datastore as _ds - if _ds.DB_PATH.exists(): - return None - state = dict(_main.MIRROR_SEED) - if not state.get("cause"): - state["cause"] = "this container has no analytical mirror yet" - state["recommendation"] = ("the boot seed runs independently of AIOS_PREWARM; if this " - "persists, check HF_TOKEN on the deployment") - return state - except Exception: # noqa: BLE001 - return None - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ THE READ-THROUGH WINDOW — owner ruling R6 ("no cap on connected-source data") via R7. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# THE PROBLEM IT REPLACES, in the owner's numbers. `core.user_tables.MAX_ROWS = 60_000` bounds a -# `ut_*` table because ALL of a tenant's tables live in ONE JSON document that `Store.get` -# deep-copies per request (D-87: four Odoo databases = 20.7 MB, ~370 ms a copy). Orders at -# **32,826** is already 55% of that ceiling; order lines (**255,286**) are 4.3× over it and GL -# lines (**971,034**) 16× over, so those two could never be grids at all — `odoo_relational.plan` -# refuses them rather than truncating, which is the correct refusal of the wrong architecture. -# -# ⭐ "NO CAP" IS NOT A BIGGER NUMBER. Raising `MAX_ROWS` would make every request slower for every -# tenant, including the ones who never open an Odoo grid. The mirror ALREADY holds all of it -# uncapped — measured on this box: 963,783 GL lines counted in ~24 ms — so the fix is to stop -# copying and serve a WINDOW: the requested slice, plus a `SELECT count(*)` that tells the truth -# about the whole. -# -# ⛔⛔ `total` IS NEVER `len(rows)`. It comes from `datastore.window`'s own count statement over -# the SAME predicate. A window whose count is its own length is a fabricated aggregate wearing an -# authoritative face ([[no-unverifiable-aggregates]]). -# -# ⛔ PREDICATES PUSH DOWN, and this is the half a client cannot be trusted with. A filter chip -# evaluated over the 200 rows that happen to be in memory would report "40 matches" out of -# 971,034 — [[one-question-two-normalizers]] at scale. `harness/filter_sql.py` compiles OUR filter -# vocabulary to SQL and is the same evaluator the TS engine is held in step with, so the fold and -# the display answer one question. -# -# ⭐ THE SPEC BELOW IS A SECOND STATEMENT OF SOMETHING `odoo_relational`'s READER ALREADY SAYS, -# AND THAT IS THE REAL RISK HERE — not SQL injection. It is gated rather than trusted: -# `verify_scopes.section_read_through` runs the REAL reader over the REAL mirror as an ORACLE and -# asserts the windowed path agrees ROW FOR ROW and TOTAL FOR TOTAL. A binding that drifts from its -# reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as -# though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]). -# -# ⚠ ONE BUCKET IS BOUND HERE (orders). The other seven still serve from the materialised copy and -# say so on the wire (`readThrough: false`), because R6's second sentence — *"if there is lag or -# it can't be done, you need to explicitly tell me why and recommend a fix"* — makes an unconverted -# grid something to REPORT, not something to leave looking converted. Adding one is a spec row plus -# a green oracle check. - -#: `ut_*` key -> how to read that grid straight out of the mirror. -#: -#: `where`/`select` are SQL WE author (never a request value); every request value is bound through -#: `params` by `filter_sql`. `cols` maps a FIELD KEY (what `odoo_relational._fields()` declares, -#: and what a saved view's filters name) to `(sql expression, coercion)`. The coercion reproduces -#: the reader's own python cast, by CALLING the reader's helpers where one exists — `_as_date` and -#: `_in_scope` are imported, not re-implemented. -#: Keys a reader does not produce (link columns, `refreshed`) are absent here on purpose: they are -#: filled by the grid at render time exactly as they are on the materialised path. -GRID_SOURCES = {} - -#: ⭐⭐ W31-T49 / C4 — THE SOURCE PROVIDERS. Odoo builds its specs below; ANY OTHER connector adds -#: its own by registering a builder here, and `_sources()` folds them into the one registry every -#: door already reads. -#: -#: ⛔ THIS IS THE SEAM THAT KEEPS "ONE REGISTRY" TRUE WHILE THE FILE STAYS ODOO-NAMED. The -#: alternative — a second `META_GRID_SOURCES` consulted beside this one — would give the platform -#: two answers to *"how is this connected grid read?"*, and every consumer (`_source_for`, -#: `whole_pool`, `population`, `odoo_tables_status`, `routes_tables.scoped_pids`) would have to -#: learn both or silently serve one. That is [[one-question-two-normalizers]] on the read path. -#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first -#: read — the same lazy rule `_sources` already follows for `odoo_relational`. -_SOURCE_PROVIDERS = [] - - -def register_source_provider(fn): - """Add a `() -> {table_key: spec}` builder to the connected-grid registry. - - Additive and idempotent by identity, so a re-import cannot double-register. Returns the number - of providers now known — a caller that wants to assert its registration took has a number. - """ - if callable(fn) and fn not in _SOURCE_PROVIDERS: - _SOURCE_PROVIDERS.append(fn) - return len(_SOURCE_PROVIDERS) - - -def _sources(): - """Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them. - - ⛔⛔ ODOO GOES THROUGH THE SEAM TOO, AND THAT IS THE POINT OF W31-T49 RATHER THAN A FLOURISH. - The first version built Odoo's specs inline here and folded other providers in afterwards, - which quietly says "Odoo is the registry and everyone else is an addendum" — two mechanisms - for one question, with the second one exercised by nobody until a connector arrives. It also - left `register_source_provider` with no production caller at all, which `verify_reachability` - correctly reddened as a capability behind no door ([[artifact-with-no-importer]]). Registering - the incumbent through its own seam makes the seam load-bearing from the first request. - - ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and - importing that module at file-import time would drag the Odoo layer into every process that - mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read. - """ - if GRID_SOURCES: - return GRID_SOURCES - for build in list(_SOURCE_PROVIDERS): - try: - extra = build() or {} - except Exception: # noqa: BLE001 - # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must - # not go dark because a newer connector's module is unhappy — and vice versa. - continue - if extra: - GRID_SOURCES.update(extra) - _ut().register_connected(*extra) - return GRID_SOURCES - - -def _odoo_sources(): - """Odoo's read-through bindings — `{table_key: spec}`. Registered as a provider below. - - ⚠ IT RETURNS a dict rather than mutating the module global: `_sources` owns the merge, so a - provider that half-built its specs and raised cannot leave a partial registry behind. - """ - rel = _rel() - # ⭐ R6 / W30-T29 — TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops - # being a fact about them. `core` never imports up, so the declaration goes this way round. - # ⚠ ALL EIGHT, not just the read-through-bound ones — every row in these tables comes from - # Odoo, which is what R6 is about; being served from the stored copy today is our conversion - # state, not a property of the data. `_sources`' own `register_connected` covers only the keys - # a provider RETURNS, so this call is not redundant with it and must not be folded into it. - _ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES]) - GRID_SOURCES = {} # the LOCAL registry this builder fills and returns - _s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)), - (lambda v: float(v or 0.0))) - GRID_SOURCES[rel.ORDERS_KEY] = { - "table": "sale_order", - # ⛔ IMPORTED, NOT RETYPED. `_CONFIRMED` is the fixed wholesale scope; if it ever changes, - # this window changes with it and the oracle check proves it did. - "where": f"{rel._CONFIRMED} AND partner_id IS NOT NULL", - "id": "id", - "needs_excluded": True, # `wholesale_scope` is resolved against the excluded set - "cols": { - "order_no": ("name", _s), - "odoo_id": ("id", _i), - "customer": ("partner_name", _s), - rel.JOIN_KEY: ("partner_id", _i), - "order_date": ("date_order", rel._as_date), - "amount_untaxed": ("amount_untaxed", _n), - "team": ("team_name", _s), - "state": ("state", _s), - "invoice_status": ("invoice_status", _s), - # ⭐ THE SCOPE COLUMN BECOMES REAL SQL, which is the point. On the materialised path it - # is `_in_scope(pid, excluded)` — a python set test, and a filter on it therefore could - # not push down. Inlining the ids (ints, from our own query) makes it a column the - # mirror can filter and sort on, so the R6 "limit" it would otherwise have earned does - # not exist. `{excluded}` is substituted by `_source_for` below. - "wholesale_scope": ("CASE WHEN partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), - }, - } - - # ═════════════════════════════════════════════════════════════════════════════════════════ - # ⭐⭐ W30-T32 — THE TWO LINE GRAINS. These are the grids R6 exists for: they have never had a - # `ut_*` table and never can, at any cap. MEASURED on this box's mirror, warm: - # sale_order_line 254,189 in the confirmed scope (256,810 unscoped) — 63.9 MB as JSON - # account_move_line 963,783 — ~240 MB as JSON - # Against `MAX_ROWS = 60_000` that is 4.2x and 16x, and against the 32 MB per-table document - # budget it is 2x and 7.5x. Read THROUGH, both serve a page in 134–166 ms. - # - # ⛔ THE FIELD KEYS BELOW ARE HALF OF A CONTRACT AND `odoo_relational` OWNS THE OTHER HALF - # (W30-T35, session E). `cols` binds a field key to SQL; the field's label, type and order are - # DECLARED THERE, once. Until that declaration lands the route answers 404 for these two keys - # (`rel.TABLES` has no entry), and `odoo_tables_status` REPORTS them as bound-not-declared - # rather than leaving them invisible — R6's second sentence applied to our own conversion. - # ⚠ A key here with no declaration there is a silent NO-CELL (`rows_from_pool` projects - # strictly); a key there with no binding here is an INACTIVE filter leaf, which WIDENS. The - # two lists are checked against each other by `verify_scopes.section_line_grids`. - ol_key = getattr(rel, "ORDER_LINES_KEY", "ut_odoo_order_lines") - gl_key = getattr(rel, "GL_LINES_KEY", "ut_odoo_gl_lines") - _ut().register_connected(ol_key, gl_key) - # ⛔ THE SCOPE IS THE ORDER'S, AND THE LINE TABLE CANNOT ANSWER IT ALONE: `sale_order_line` - # carries no `state` (12 columns, measured), so the confirmed-order scope — and `order_date`, - # and the order NAME a person reads the grid by — only exist across the join. MEASURED, warm, - # best of two: the JOIN beats `order_id IN (SELECT id FROM sale_order WHERE …)` at both depths - # (166 / 483 ms against 237 / 565 ms at offset 0 / 200,000), so the shape is chosen on a - # number rather than on taste. - # ⚠ `_CONFIRMED` is IMPORTED and QUALIFIED, never retyped — it opens with the bare column - # `state`, which `sale_order_line` does not have, so the prefix is what keeps it unambiguous - # if that table ever gains one. `section_line_grids` counts the same population a second way - # (a subquery, not a join) and the two must agree, which is what catches a mis-qualification. - GRID_SOURCES[ol_key] = { - "from_sql": "(sale_order_line sol JOIN sale_order so ON so.id = sol.order_id)", - "tables": {"sol": "sale_order_line", "so": "sale_order"}, - "where": f"so.{rel._CONFIRMED}", - "id": "sol.id", - "needs_excluded": True, - "cols": { - "odoo_id": ("sol.id", _i), - "order_no": ("so.name", _s), - "order_id": ("sol.order_id", _i), - "customer": ("sol.order_partner_name", _s), - rel.JOIN_KEY: ("sol.order_partner_id", _i), - "product": ("sol.product_name", _s), - rel.PRODUCT_JOIN_KEY: ("sol.product_id", _i), - "qty": ("sol.product_uom_qty", _n), - "price_subtotal": ("sol.price_subtotal", _n), - "margin": ("sol.margin", _n), - "purchase_price": ("sol.purchase_price", _n), - "order_date": ("so.date_order", rel._as_date), - "state": ("so.state", _s), - "wholesale_scope": ( - "CASE WHEN sol.order_partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), - }, - } - # ⚠ UNSCOPED ON PURPOSE, and it is a decision rather than an omission: every other grid here - # carries a fixed scope, but a GENERAL LEDGER whose draft and cancelled entries are invisible - # is a ledger that cannot be reconciled. `parent_state` rides as a column so a person filters - # in SQL over all 963,783 rows instead of us choosing for them. (Posted-only is 944,846.) - # The join to `account_account` is what makes `account_code` — the key `ut_odoo_accounts` is - # linked on — available at all; the mirror flattens `account_id`/`account_name` onto the line - # but not the CODE, and 192 accounts hash-join for free. - GRID_SOURCES[gl_key] = { - "from_sql": ("(account_move_line aml LEFT JOIN account_account aa " - "ON aa.id = aml.account_id)"), - "tables": {"aml": "account_move_line", "aa": "account_account"}, - "where": "", - "id": "aml.id", - "needs_excluded": True, - "cols": { - "odoo_id": ("aml.id", _i), - "entry": ("aml.move_name", _s), - "move_id": ("aml.move_id", _i), - "account": ("aml.account_name", _s), - rel.ACCOUNT_JOIN_KEY: ("aa.code", _s), - "customer": ("aml.partner_name", _s), - rel.JOIN_KEY: ("aml.partner_id", _i), - "date": ("aml.date", rel._as_date), - "debit": ("aml.debit", _n), - "credit": ("aml.credit", _n), - "balance": ("aml.balance", _n), - "line_type": ("aml.display_type", _s), - "move_type": ("aml.move_type", _s), - "parent_state": ("aml.parent_state", _s), - "wholesale_scope": ( - "CASE WHEN aml.partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), - }, - } - # ⭐ W30-T31 — THE GL ACCOUNT REGISTRY, BOUND BECAUSE IT IS THE ONE GRID THAT CAN ACTUALLY - # STOP MATERIALISING TODAY. 192 rows, nothing folds it, nothing links at it, and it fits - # inside one window — the three conditions `_unmaterialisable` checks. It is small, and that - # is the point: it is the first shipped database whose rows are NOT in the tenant document, - # so the stratum is proven on a real table instead of on a mechanism with no subject. - # ⚠ `account_fields()` declares six columns and `read_accounts` is one `cur.execute` over - # `account_account`; `section_read_through`'s differential oracle holds this binding to it. - # ⚠ `is_expense` REPRODUCES `read_accounts`' predicate IN SQL rather than inventing one, and - # that predicate is itself a copy of the semantic layer's `gl_lines` scope. Three statements of - # one rule is two too many, but the reader's own comment explains why it is copied rather than - # imported, and `section_read_through`'s differential oracle is what keeps this one honest. - GRID_SOURCES[rel.ACCOUNTS_KEY] = { - "table": "account_account", - "where": "", - "id": "id", - "cols": { - rel.ACCOUNT_JOIN_KEY: ("code", _s), - "account_name": ("name", _s), - "odoo_id": ("id", _i), - "account_type": ("account_type", _s), - "is_expense": ("CASE WHEN account_type IN ('expense','expense_depreciation') " - "THEN '1' ELSE '' END", _s), - }, - } - return GRID_SOURCES - - -#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers -#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`, -#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid. -register_source_provider(_odoo_sources) - - -def sync_read_through(cur=None): - """Register every grid that may stop storing rows, and REPORT why the rest may not. - - Returns `({key: eligible}, {key: reason})`. Idempotent, cheap, and safe to call from any door: - registration is additive and `strip_materialised` is a no-op once a table is empty. - - ⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility - question needs BOTH halves — `odoo_relational`'s field declarations (for the fold matrix) and - the mirror (for the size) — and `core` may import neither. - """ - eligible, reasons = _unmaterialisable(cur) - keys = [k for k, ok in eligible.items() if ok] - if keys: - _ut().register_read_through(*keys) - return eligible, reasons - - -#: ⚠ A mirror can be `ready()` and still lack a column (`ready()` reads entity PHASES; column -#: backfills checkpoint separately) — the gap that already cost a live 500. Every projected -#: expression is checked against the real column list and degraded to a literal, exactly as -#: `odoo_relational._col` does for the reader, so a fresh Space serves a blank cell rather than a -#: DuckDB Binder error. -#: -#: ⛔ W30-T32 — IT TAKES AN ALIAS MAP NOW, AND WITHOUT THAT THE GUARD WAS ABOUT TO GO BLIND. The -#: line-grain grids project `sol.price_subtotal` / `aml.parent_state`, and a dotted string is not -#: `isalnum()`, so the old single-table version returned EVERY qualified expression unchecked — -#: the same "expression, nothing to check" branch that correctly skips a CASE. Both of those -#: columns are 2026-07-28 backfills that a mirror can genuinely be missing, so the blind spot -#: would have surfaced as a bare DuckDB Binder error on a fresh Space, which is precisely the -#: failure this helper exists to prevent ([[gate-answers-the-wrong-question]]). -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W30-T31 / D-87 — WHICH CONNECTED GRIDS MAY STOP MATERIALISING, AND WHY MOST MAY NOT YET. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The prize is real: all of a tenant's `ut_*` tables live in ONE document that `Store.get` -# deep-copies on EVERY call, hit or miss, and the four original Odoo grids are 20.7 MB / ~370 ms -# of it. Every permission check in the app pays that. -# -# ⛔ AND YOU CANNOT SIMPLY DELETE THE ROWS, WHICH IS THE FINDING THIS FUNCTION ENCODES. Measured -# mechanically across the eight field declarations, not by eye: -# * `ut_odoo_invoices` and `ut_odoo_orders` — 16.2 of those 20.7 MB — are folded by SIX link -# rollups on `ut_odoo_customers` (`ar_outstanding`, `open_invoices`, `oldest_due`, -# `invoiced_all_time`, `order_count`, `last_order`). `automation_engine.compute_relation_cells` -# answers those by reading the RAW document, so with the rows gone it writes zeros — silently. -# * every table with a `link` pointing AT it (agents, vendors, bills, customers) has its link -# CELLS materialised the same way, from the target's stored rows. -# * `ut_odoo_customers` and `ut_odoo_products` carry SOURCE rollups, and `rollup_sql.compute` -# writes those cells INTO their own stored rows: no rows, no cells. -# * and while the grid client still asks for a whole table (F's W30-T42 is what changes this), -# a population larger than one window could only be served by TRUNCATING it — which R6 -# forbids more strongly than it forbids a cap. -# -# ⭐ SO THE PREDICATE IS DERIVED FROM THE DECLARATIONS RATHER THAN LISTED. The day a lane converts -# those six link rollups to SOURCE rollups (`orders_ytd` on that same table is the precedent), or -# the day the client pages, the affected grids become eligible here with NO code change — and -# until then each one's reason is reported per table on the status door, which is R6's second -# sentence applied to our own conversion state. -def _fold_reasons(rel): - """`{table_key: "why its stored rows are still read by something else"}`. - - Read out of the FIELD DECLARATIONS themselves — one pass over `rel.TABLES`. A table absent - from this map is folded by nothing. - """ - reasons = {} - - def _add(key, why): - reasons.setdefault(str(key), []).append(why) - - for _bucket, key, _label, mk in rel.TABLES: - try: - fields = mk() - except Exception: # noqa: BLE001 - continue - by_key = {f.get("key"): f for f in fields if isinstance(f, dict)} - short = str(key).replace("ut_odoo_", "") - for f in fields: - if not isinstance(f, dict): - continue - if f.get("type") == "link" and (f.get("link") or {}).get("table"): - _add(f["link"]["table"], f"{short}.{f['key']} is a link whose cells are built " - f"from these rows") - if f.get("type") != "rollup": - continue - bag = f.get("rollup") or {} - if isinstance(bag.get("source"), dict): - _add(key, f"{short}.{f['key']} is a source rollup and its cells are written " - f"into these rows") - continue - tgt = ((by_key.get(str(bag.get("link") or "")) or {}).get("link") or {}).get("table") - if tgt: - _add(tgt, f"{short}.{f['key']} folds these rows") - return {k: "; ".join(v) for k, v in reasons.items()} - - -def _unmaterialisable(cur=None): - """`({key: eligible}, {key: reason})` — who may stop storing rows, and why the rest may not. - - ⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in - the first cut of this function. A table too big for the tenant document has NO materialised - option at all, so making it ineligible would have handed `odoo_relational.plan` a `row_limit` - of None and invited it to build 963,783 python dicts. The window ceiling is a different, much - softer thing: it only limits what the whole-table CLIENT door can serve today. - - rows > MAX_ROWS -> read-through REQUIRED (the document cannot hold it) - else if something folds it -> stays materialised (a fold over no rows writes ZEROS) - else if rows > WINDOW_MAX -> stays materialised until the client pages (W30-T42) - else -> eligible - - ⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never - part of the materialised spawn is eligible — a table is never freed by a question we skipped. - """ - rel = _rel() - folds, eligible, reasons = _fold_reasons(rel), {}, {} - # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT. - # W30-T35 declared the two line grains, which have no python row builder anywhere — their - # absence from `_READERS` IS that statement — so a table the spawn could never materialise - # started reading as spawned, and on a mirror-less deployment fell through to "its size could - # not be read" and came back INELIGIBLE. That inverts this function's own first law (size - # forces read-through; it never blocks it). Ask the question through the reader map, which is - # what actually decides whether a row could ever have been built. - spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})} - from harness import datastore - ut = _ut() - for key, spec in _sources().items(): - total = None - if cur is not None: - try: - frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") - else {"table": spec["table"]}) - total = datastore.window(select="1", where=spec.get("where") or "", - order_by=spec["id"], limit=1, cur=cur, **frm)["total"] - except Exception: # noqa: BLE001 - total = None - if total is not None and total > ut.MAX_ROWS: - eligible[key] = True # no other home exists; this is not a choice - continue - if total is None and key not in spawned: - eligible[key] = True # never materialised, so the mirror is its home - continue - why = [] - if total is None: - why.append("its size could not be read from the mirror on this deployment") - if key in folds: - why.append(folds[key]) - if total is not None and total > datastore.WINDOW_MAX: - why.append(f"its {total:,} rows exceed the {datastore.WINDOW_MAX:,}-row window and " - f"the grid client still asks for whole tables, so un-materialising it " - f"today could only truncate") - eligible[key] = not why - if why: - reasons[key] = "; ".join(why) - return eligible, reasons - - -def _degrade(expr, have_by_alias, default_alias=""): - bare = expr.strip() - alias, _, col = bare.partition(".") - if col and alias.replace("_", "").isalnum() and col.replace("_", "").isalnum(): - have = have_by_alias.get(alias) - if have is None: - return expr # an alias we do not own — leave it to the caller's SQL - return expr if col.lower() in have else "NULL" - if not bare.replace("_", "").isalnum(): - return expr # an expression, not a bare column — nothing to check - # ⚠ A BARE COLUMN IN A JOINED SPEC RESOLVES THE WAY SQL RESOLVES IT — against every table in - # the FROM, not against nothing. Checking it against `have_by_alias[""]`, which a joined spec - # does not have, would degrade every such column to NULL: a silent blank cell, which is the - # failure this helper exists to avoid rather than to cause. - have = have_by_alias.get(default_alias) - if have is None: - have = set().union(*have_by_alias.values()) if have_by_alias else set() - return expr if bare.lower() in have else "NULL" - - -def _source_for(cur, table_key): - """The resolved spec for one connected grid, or None when this grid is not read-through yet.""" - spec = _sources().get(str(table_key or "")) - if not spec: - return None - from harness import datastore - # ⚠ `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table - # spec declares none and its columns are bare, so it degrades against `table` as before. - aliases = dict(spec.get("tables") or {}) - if spec.get("table"): - aliases.setdefault("", spec["table"]) - have_by_alias = {} - for alias, tname in aliases.items(): - cols = datastore.columns_of(tname, cur=cur) - if not cols: - return None # the mirror has no such table on this deployment - have_by_alias[alias] = cols - excluded = "" - if spec.get("needs_excluded"): - ids = sorted(int(p) for p in _rel().excluded_ids(cur)) - # `-1` keeps the IN-list non-empty and matches no Odoo id, so the SQL shape is constant - # whether or not this tenant excludes a channel. - excluded = ", ".join(str(i) for i in ids) or "-1" - cols = {} - for key, (expr, cast) in spec["cols"].items(): - cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr, - have_by_alias), cast) - return {**spec, "cols": cols} - - -#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in -#: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of -#: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of -#: the person BEFORE the wait rather than after it. -_DEEP_PAGE = 100_000 - - -#: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our -#: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid -#: cannot disagree about whether a column is text or a number. -_FILTER_TYPE = {"select": "status", "checkbox": "text"} - - -def _filter_columns(spec, fields): - """`{colId: {sql, type, aggregate}}` — what `filter_sql` needs to compile a predicate. - - Only columns with a real mirror expression are offered. An omitted column is UNKNOWN to the - compiler, which skips it — and that is the one behaviour that must be REPORTED rather than - accepted, because an ignored condition WIDENS (the tri-state engine's inactive-leaf rule). - `_unpushable` below turns every such skip into an R6 sentence. - """ - by_key = {f["key"]: f for f in fields} - out = {} - for key, (expr, _cast) in spec["cols"].items(): - ftype = str((by_key.get(key) or {}).get("type") or "text") - out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False} - return out - - -def _leaf_cols(nodes): - """Every `colId` a filter tree names, at any depth.""" - seen = set() - for n in (nodes or []): - if not isinstance(n, dict): - continue - if n.get("children") is not None: - seen |= _leaf_cols(n.get("children")) - elif n.get("colId"): - seen.add(str(n["colId"])) - rhs = n.get("rhs") - if isinstance(rhs, dict) and rhs.get("colId"): - seen.add(str(rhs["colId"])) - return seen - - -def _json_arg(raw, what): - """Decode a JSON query argument, or refuse. ⛔ NEVER degrade to "no filter": a filter that - silently fails to parse WIDENS the answer, and the caller sees a plausible bigger number.""" - if raw in (None, ""): - return None - try: - val = json.loads(raw) - except Exception: # noqa: BLE001 - raise err(400, "bad_argument", f"{what} must be JSON") - return val - - -@router.get("/odoo-tables/{table_key}/rows") -def odoo_table_rows(table_key: str, - offset: int = Query(default=0, ge=0), - limit: int = Query(default=0), - filters: str = Query(default=None), - filterConj: str = Query(default="and"), - sorts: str = Query(default=None), - search: str = Query(default=None), - session: Session = Depends(require_session)): - """ONE WINDOW over a connected grid, read straight from the mirror (contract C2). - - `{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` — - `rows` is the requested slice and `total` is the count of everything the CURRENT PREDICATE - matches, from its own `SELECT count(*)`. `rows.length < total` is the normal case. - - ⛔ `total` is never `len(rows)`; `totalUnfiltered` is the population with the predicate - dropped, so a client can render "N of M" without inventing either number. - ⛔ The filter, the sort and the search all resolve in SQL against the whole table. Anything - that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in - `limits` with its cause and a recommendation — R6's second sentence — and never silently - dropped, because an ignored condition widens. - """ - import aios_grid - from harness import datastore - from routes_tables import _defn_or_refuse - - rel = _rel() - # THE WALL FIRST, and it is the SAME one the materialised path uses — 404 for a key that does - # not exist, 403 for one this session may not open. Reused rather than re-stated: a second - # idea of "may this session open this database" is a permission bug waiting to happen. - # - # ⭐⭐ W33-T02 / D-213 — `defs_only=True`, AND THIS ROUTE IS THE CLEANEST OPT-IN ON THE BOARD. - # `defn` is read EXACTLY ONCE below, for `recordMode`; the fields come from `rel.TABLES`'s - # `mk_fields()` and the rows from `datastore.window`, so nothing here has ever touched - # `defn["rows"]`. That is what makes it safe by inspection rather than by argument — and it is - # why the same change must NOT be swept across this file: `odoo_tables_status` a few hundred - # lines down reads `table["rows"]` and its per-row `refreshed` stamps for the MATERIALISED - # grids, and a projection there raises `KeyError` on every one of them. - # ⚠ `mirror_stamp`'s bare `except: return ""` would convert exactly that raise into a silently - # BLANK `refreshed` column rather than a red — which is why the boundary is drawn here, at the - # one function that provably needs no row, instead of at the file. - defn = _defn_or_refuse(session, table_key, defs_only=True) - fields = None - for _bucket, key, _label, mk_fields in rel.TABLES: - if key == table_key: - fields = mk_fields() - break - if fields is None: - raise err(404, "not_connected", "that database is not a connected Odoo grid") - - # ⭐⭐ W31-T45 / D-169 — THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot. - # That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process - # has open the one this TENANT's rows live in". A definition wall is satisfied the moment a - # tenant's own document declares a connected table — which is exactly what R2's Meta Ads spawn - # gives GTM Lab — and it would then serve that session a window over whatever DuckDB file the - # process happens to be pinned to. Rows, not counts. So this door RAISES. - # ⛔ 409, NOT 503, and the two were one line from being confused: `DatastoreMismatch` subclasses - # `RuntimeError`, and the handler directly below turns a `RuntimeError` into - # `503 store_not_ready` — "the store is completing its first sync, retry in a few minutes". A - # mis-pinned process never becomes un-mis-pinned by waiting, so relaying it as a retry would - # turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the - # try, not inside it. - try: - cur, mismatch = _mirror_cur(session) - except RuntimeError as e: - # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the - # mismatch subclass, which is what makes this broad clause safe to keep here. - raise err(503, "store_not_ready", str(e)) - if mismatch: - raise err(409, "cross_tenant_store", mismatch) - - spec = _source_for(cur, table_key) - if spec is None: - raise err(409, "not_read_through", - "this connected database is still served from its stored copy; it has no " - "read-through binding on this deployment yet") - - cols = _filter_columns(spec, fields) - limits, tree = [], _json_arg(filters, "filters") - sort_spec = _json_arg(sorts, "sorts") or [] - - # ── the predicate, pushed down ──────────────────────────────────────────────────────────── - named = _leaf_cols(tree) - missing = sorted(named - set(cols)) - if missing: - limits.append({ - "subject": ", ".join(missing), "effect": "filter_ignored", - "cause": "these columns have no expression in the mirror (a link, a rollup, or a " - "column this deployment's mirror has not backfilled), so a condition on " - "them cannot be answered in SQL", - "recommendation": "filter on the id column the link is built from, or open the " - "linked database directly"}) - # ⭐ R6, MEASURED, AND IT IS THE KIND OF LIMIT THE RULING EXISTS FOR — a difference in the - # ANSWER, not in the speed. - # - # `filter_sql._value_sql` compiles every numeric comparison as `round_even(x, 0)`, and says - # why: *"reproduces `aios_grid._round` … The grid displays rounded values; filters must agree - # with what is on screen."* That is true of `source: "odoo"` columns, which `rows_from_pool` - # rounds. It is FALSE here — these columns are `source: "overlay"` (a storage choice, not a - # display one) so `rows_from_pool` passes the exact value through, and the TS engine - # (`useVisibleRows.toNum`) does not round either. So the pushdown compares at whole units while - # the cell beside it carries cents. - # - # MEASURED on the live orders mirror (32,700 rows, 47.9% with a non-integer amount): - # > 1000 python 4,476 vs SQL 4,473 (-3) - # > 173.6 python 24,711 vs SQL 24,726 (+15) - # > 500.25 python 11,028 vs SQL 11,026 (-2) - # Small, and NOT nothing. Rounding the wire to match would have been the other fix and it was - # rejected on measurement: it changes 96 of every 200 money cells (173.55 -> 174) to remove a - # 0.05% counting difference — a visible product regression traded for an invisible one. - # ⛔ SO IT IS REPORTED INSTEAD. Booked for the owner of `harness/filter_sql.py`, which is not - # this fence; see mailbox/D.md. - numeric = sorted(k for k in (named & set(cols)) - if cols[k]["type"] in ("currency", "int", "pct")) - if numeric: - limits.append({ - "subject": ", ".join(numeric), "effect": "precision", - "cause": "a number condition is evaluated in SQL at whole-unit precision " - "(`filter_sql` rounds to match the grids whose values the server rounds), " - "while these cells carry their exact value — so a row within half a unit of " - "the threshold can fall on the other side of it", - "recommendation": "compare against a whole number, or use a range that does not sit " - "on a fractional boundary"}) - - where, params = spec["where"], [] - try: - pred = _fs().compile_filter_tree( - tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols, - today=_today()) - except ValueError as e: - # ⛔ A REFUSAL IS AN ANSWER, NOT A CRASH — and it must not become "no filter". Ranking ops - # ("top 10") have no SQL form in this compiler; saying so beats returning every row. - raise err(400, "filter_unsupported", str(e)) - if pred is not None: - if pred.uses_aggregate: - raise err(400, "filter_unsupported", - "a condition on an aggregate column belongs in HAVING, and that path is " - "deliberately not built for windowed grids") - where = f"({where}) AND {pred.sql}" if where else pred.sql - params.extend(pred.params) - if str(search or "").strip(): - got = _fs().compile_search(search.strip(), cols) - if got is not None: - where = f"({where}) AND {got.sql}" if where else got.sql - params.extend(got.params) - - # ── the order, made TOTAL ───────────────────────────────────────────────────────────────── - # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one - # row on two pages and drop another entirely — a duplicate the user sees with no error - # anywhere. `compile_order_by`'s own docstring says so. - order = _fs().compile_order_by(sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"] - - select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) - # ⚠ EXACTLY ONE of `table`/`from_sql` — `window` raises if both or neither arrive, so the - # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM. - frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', - where=where, params=tuple(params), order_by=order, - offset=offset, limit=(limit or None), cur=cur, **frm) - if win["clamped"]: - limits.append({ - "subject": "limit", "effect": "window_clamped", - "cause": f"one response carries at most {datastore.WINDOW_MAX} rows so a single " - f"request cannot exhaust memory for every other tenant on the process", - "recommendation": "page with `offset`; `total` already reports the whole population, " - "and no row is unreachable"}) - # ⭐ R6's SECOND SENTENCE, ON THE ONE LIMIT THAT SURVIVES THE CONVERSION. Removing the row cap - # does not make every row equally cheap: `LIMIT/OFFSET` WALKS the offset, so the deeper the - # page the longer the scan. MEASURED warm on 963,783 GL lines — offset 0: 134 ms · 100,000: - # 71 ms · 900,000: **1,450 ms**; sorted by date rather than by id, offset 500,000: 1,645 ms. - # It is a real cost, it is nobody's mistake, and the owner asked to be told rather than to - # discover it: say so with the fix, which is a CURSOR the client has to send. - if offset >= _DEEP_PAGE: - limits.append({ - "subject": "offset", "effect": "slow", - "cause": f"a page {offset:,} rows deep is reached by walking every row before it " - f"(SQL OFFSET has no other meaning), which costs about a second past " - f"half a million rows", - "recommendation": "jump with a filter or a sort instead of scrolling, or ask for " - "keyset paging (`id > `), which is O(page) at any depth " - "and needs the client to send the last row it holds"}) - - # ── the wire rows, through the SAME serialiser the materialised path uses ────────────────── - # ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the - # `refreshed` column identically instead of one of them leaving it blank. - rows_src = _pool_from_window(spec, win, - stamp=mirror_stamp(spec, cur, table_key, - session.runtime)) - overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src} - - # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID. - # - # A user-added column on a connected grid has nowhere per-user to live: there is no `ut_*` row - # to hang it on any more, and `table_store`'s per-user strata would make a SHARED view name a - # column other accounts do not have — an unknown column is an INACTIVE condition in the - # tri-state engine, which WIDENS. `core/shared_overlay.py` was built for exactly this and has - # had no product door since it shipped (W29-T62). - # - # ⛔ `cells(table_key, pids)` TAKES THE PIDS AND THERE IS NO "EVERYTHING" CALL — and here that - # is an asset rather than a chore: **the window IS the scoped row set**, already narrowed by - # the wall and the predicate, so the argument it demands is the list we just fetched. - # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already - # answered "may this session open this surface". - shared_defs = _so().fields(table_key, st=session.runtime) - if shared_defs: - fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()] - for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src], - st=session.runtime).items(): - overlays.setdefault(pid, {}).update(cells) - rows = aios_grid.rows_from_pool(rows_src, fields, overlays) - - unfiltered = win["total"] if where == spec["where"] else datastore.window( - select="1", where=spec["where"], order_by=spec["id"], limit=1, cur=cur, **frm)["total"] - - return {"ok": True, "fields": fields, "rows": rows, - # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`. - "total": win["total"], "totalUnfiltered": unfiltered, - "offset": win["offset"], "limit": win["limit"], "limits": limits, - "today": _today(), "identity": {"pid": "pid"}, - "scope": {"table": table_key, "readThrough": True}, - "recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)} - - -def mirror_stamp(spec, cur=None, table_key="", rt=None): - """⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date. - - ⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table - declares `refreshed` (`odoo_relational._refreshed_field`), and a MATERIALISED grid gets a - per-row stamp written by the spawn. A read-through grid has no stored row to carry one, so - `_pool_from_window` produced no `refreshed` key at all and `rows_from_pool` projected it as - `""` — a freshness column that used to say something and now silently says nothing, on the two - biggest grids in the product. Nothing goes red for that; it just quietly stops being true. - - ⭐ THE HONEST VALUE ALREADY EXISTS — the mirror keeps a per-entity sync stamp in `_sync_state`, - which is the same fact one level up: these rows are as fresh as the last sync of the table - they are read through. It is per-TABLE, not per-row, and that is not a downgrade — for a grid - that stores no rows, "when did this table last sync" IS the per-row answer. - - ⚠ READ ON THE CALLER'S CURSOR, never `datastore.status()`. That opens a SECOND connection to a - file DuckDB holds exclusively, so a freshness column would turn a working grid into an - IOException — a cosmetic fix taking out the feature it decorates. - ⚠ Date only (`YYYY-MM-DD`), because `refreshed` is declared `type: "date"` and the grid's date - renderer is what reads it; handing it a full timestamp renders the ISO string a user should - never see (the wave-26 `copyData` scar, one column over). - - ⛔⛔ THE TABLE COMES FROM THE **ID COLUMN'S ALIAS**, AND THE FIRST DRAFT GOT THIS WRONG IN THE - ONLY WAY THAT MATTERS. It read `spec["table"]` with a `tables[""]` fallback — and **neither of - the two grids D-155 is about has a bare `table` key**: `ut_odoo_order_lines` and - `ut_odoo_gl_lines` are join-shaped (`from_sql` + `tables: {"sol": …, "so": …}` / - `{"aml": …, "aa": …}`), so the lookup returned `""` and the fix would have shipped doing - nothing on exactly the two grids it was written for — green gate, unchanged product. It was - caught only because the gate's fixture used `ut_odoo_orders`, a MATERIALISED grid that never - had the defect [[gate-answers-the-wrong-question]]. - ⭐ The id column IS the grain: `sol.id` means this grid is one row per `sale_order_line`, and - the freshness of a joined lookup table (`sale_order`, `account_account`) is not this grid's - freshness. So the alias is read off `spec["id"]`, and a bare `id` degrades to `table` — which - is exactly the single-table case. - - ⛔ AND IT ANSWERS `""` FOR A **MATERIALISED** GRID, ON PURPOSE. Those eight tables store a - per-ROW `refreshed` written by the spawn, which is strictly better than one table-level date — - and this value arrives as an OVERLAY, so returning a stamp here would quietly overwrite eight - working grids' per-row stamps while fixing two blank ones. D-155's subject is the grid that has - NOWHERE to keep a row; a fix that also rewrites the grids that do is a different, unrequested - change [[reuse-and-delete-are-hypotheses]]. - """ - # the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot - # tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank), - # never a stamp that might overwrite a per-row one. - if table_key: - try: - if _ut().materialises(str(table_key), st=rt): - return "" - except Exception: # noqa: BLE001 - return "" - ident = str(spec.get("id") or "") - alias = ident.split(".", 1)[0] if "." in ident else "" - table = str((spec.get("tables") or {}).get(alias) or spec.get("table") - or (spec.get("tables") or {}).get("") or "") - if not table or cur is None: - return "" - try: - row = cur.execute("SELECT updated_at FROM _sync_state WHERE entity = ?", - [table]).fetchone() - except Exception: # noqa: BLE001 - return "" # no mirror bookkeeping ⇒ blank, exactly as before - return str((row or [""])[0] or "")[:10] - - -def _pool_from_window(spec, win, stamp=""): - """`[{pid, **cells}]` — THE one place a mirror window becomes product rows. - - ⛔ ONE BUILDER, TWO CALLERS, and that is deliberate rather than tidy: the windowed route and - the whole-table read-through below would otherwise each cast the same columns their own way, - and a cell that renders differently depending on which door served it is this repo's recorded - defect class ([[one-question-two-normalizers]]). - - ⚠ `stamp` (D-155) rides here for that same reason: both doors must produce the same - `refreshed` cell, and a caller that forgot it would give one door a freshness column and the - other a blank one. Empty when the caller cannot cheaply know — blank is what shipped, so the - degradation is the previous behaviour rather than a new wrong value. - """ - keys = list(spec["cols"]) - extra = {"refreshed": stamp} if stamp else {} - return [{"pid": int(r[0]), **extra, - **{k: spec["cols"][k][1](v) for k, v in zip(keys, r[1:])}} for r in win["rows"]] - - -class TooBigToMaterialise(Exception): - """A read-through table asked for WHOLE exceeds one window — refuse, never truncate.""" - - -def population(table_key, cur=None, rt=None): - """How many rows a read-through grid HAS, without fetching one — or None if it is not bound. - - ⭐⭐ W31 / B's ask (mailbox/B.md B-2). `routes_tables.scoped_pids` learned "this grid is too big - to list" the only way that existed: call `whole_pool()` and catch `TooBigToMaterialise`. That - pulls a full 5,000-row window out of DuckDB and throws it away on every `/workspace` for - `ut_odoo_gl_lines` and `ut_odoo_order_lines` — MEASURED by B at ~1,600 ms of a ~3,460 ms - in-proc envelope. This asks the SIZE instead: one `SELECT count(*)` over the same predicate. - - ⛔ IT ANSWERS A DIFFERENT QUESTION, NOT THE SAME ONE MORE CHEAPLY, and that distinction is what - keeps it safe. B's constraint is that the pid set must stay IDENTICAL to `scoped_pool`'s BY - CONSTRUCTION rather than by two queries that agree today ([[one-question-two-normalizers]]) — - so this returns a COUNT and never a pid. A caller uses it to decide whether to ask for pids at - all; when it does ask, the pids still come from the one `whole_pool` fetch they always did. - - ⚠ `None` means "no read-through binding on this deployment", which is NOT `0`. A caller that - collapses them reports an empty grid where it should report an unbound one — the exact - distinction `odoo_tables_status` already reports as `bound_not_declared`. - ⚠ `total` is the count over the spec's OWN `where`, i.e. the same population `whole_pool` - would return — the fixed scope is not dropped for being cheaper. - """ - from harness import datastore - if rt is not None: - rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry - cur = cur if cur is not None else datastore.ro_con() - spec = _source_for(cur, table_key) - if spec is None: - return None - frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"], - limit=1, cur=cur, **frm)["total"] - - -def whole_pool(table_key, cur=None, rt=None): - """Every row of a read-through grid, in `scoped_pool`'s shape — or a refusal. - - ⚠ THIS EXISTS FOR THE CLIENT WE HAVE, NOT THE ONE WE WANT. The grid still fetches whole - tables (`GET /tables/{key}/rows`); paging is F's W30-T42. So a database whose rows have left - this tenant's document has to be servable to that client somehow, and the honest answer for a - small one is "read all of it from the mirror". `_unmaterialisable` only ever registers a table - that fits, so the refusal below is a guard against the population GROWING past the window - later — at which point R6 requires a sentence, not a quietly shorter grid. - - ⭐⭐ W31-T45 / D-169 — `rt` IS THE TENANT RUNTIME, AND IT IS OPTIONAL FOR A STATED REASON - RATHER THAN A LAZY ONE. This function serves ROWS, so it is the door where a cross-tenant read - would be worst — but its only production caller is `routes_tables.py::_read_through_rows`, - which is in another lane's fence THIS WAVE (B's W31-T20 is rewriting that exact path) and does - not thread a session down to here. Making `rt` required would break that caller on import; a - second predicate invented locally would be one question with two normalizers, a recorded - defect class here. So the guard fires when a caller passes the runtime it already holds, and - the ask to thread `session.runtime` through `_read_through_rows` is posted to B in - `mailbox/D.md`. ⛔ Until that lands this door is guarded only by `_defn_or_refuse` upstream — - stated here rather than left for a reader to discover, because an unguarded row door is - precisely what D-169 is about. - """ - from harness import datastore - if rt is not None: - cur = cur if cur is not None else _rt().mirror_cursor(rt) - rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent - cur = cur if cur is not None else datastore.ro_con() - spec = _source_for(cur, table_key) - if spec is None: - raise TooBigToMaterialise( - f"{table_key}: this database is served through the mirror but has no read-through " - f"binding on this deployment, so its rows cannot be read at all") - frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) - win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=spec.get("where"), - order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm) - if win["total"] > len(win["rows"]): - # ⛔ REFUSE, NEVER TRIM. A short grid that says nothing is exactly the silent truncation - # R6's second sentence is about — and the caller turns this into a message with a cause. - raise TooBigToMaterialise( - f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row " - f"window, and this database no longer stores rows in the tenant document; it can " - f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)") - return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt)) - - -def _fs(): - from harness import filter_sql - return filter_sql - - -def _so(): - import core.shared_overlay as shared_overlay - return shared_overlay - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _today(): - import time - return time.strftime("%Y-%m-%d") +"""routes_odoo_tables.py — the door to the Odoo relational spawn (wave 27 item 17, contract C8). + +Two endpoints and no cleverness: refresh the locked Odoo databases, and report their freshness. +The work itself lives in `odoo_relational.py`; this file only decides WHO may ask and turns a +refusal into a status code. + +⛔ THE MOUNT IS DONE (`main.py` includes this router) and the wave-23 scar it was written against +— three finished routers shipping 404-dead behind green gates — is covered by `verify_api` +enumerating `main.app.routes`. ⚠ BUT WAVE 28 PROVED THAT CONTROL IS ONLY HALF OF THE QUESTION: +being mounted is not being CALLABLE. This router was mounted, enumerated, green, and answered a +plain-text 500 to every request for a day because of an attribute typo in the admin check +(`session.is_admin`, which does not exist). A route-existence check cannot see that; only calling +it can. `verify_api` now does both. + +ADMIN-GATED, and not for tidiness: a refresh REWRITES four locked databases for the whole tenant +and deletes the rows that left the population. That is an operator action. +⭐ WAVE 30 (R6/R7, contract C2) ADDS A THIRD ENDPOINT AND IT IS A DIFFERENT KIND OF THING: the +READ-THROUGH WINDOW. The two above operate on the materialised copy; `/{table_key}/rows` does not +read the copy at all. See the block above `GRID_SOURCES` for why that had to change. +""" +import json + +from fastapi import Depends +from fastapi import APIRouter, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +def _rel(): + import odoo_relational + return odoo_relational + + +def _rt(): + from harness import runtime + return runtime + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W31-T45 / D-169 — THE CROSS-TENANT MIRROR GUARD, AND WHY EVERY DOOR BELOW GOES THROUGH IT. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# `harness/datastore` holds ONE process-wide DuckDB connection over a process-global `DB_PATH`, +# and ONE Space process serves EVERY tenant. `TenantRuntime.assert_datastore_matches` was written +# for exactly the failure that arrangement produces — its own docstring names it: *"an analytical +# read served from another tenant's file returns real, plausible, wrong rows … and the number +# reconciles against the wrong book"* — and until this wave its only caller anywhere was +# `verify_api.py`. The guard was correct, tested, and governed nothing. +# +# ⛔ WHAT ACTUALLY HELD THE BOUNDARY WAS THE CUSTOMER LIST, NOT THE CODE. Three of the four doors +# below are reachable only by a tenant whose own `user_tables` document declares the table +# (`_defn_or_refuse`) or whose slug passes `is_royal` — so no second tenant could reach the mirror +# because no second tenant had mirror databases. R2 puts Meta Ads on this same mirror for GTM Lab, +# which ends that fact. A boundary that holds because of who our customers are is not a boundary. +# +# ⚠ AND ONE DOOR WAS ALREADY THROUGH IT TODAY — measured, not theorised: `/odoo-tables/status` +# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable` +# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row +# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes. +def _mirror_cur(session): + """A mirror cursor for THIS session's tenant, or a refusal that says which failure it is. + + Returns `(cursor, mismatch_sentence)` — exactly one of the two is None, so a caller cannot + accidentally treat "refused" as "empty". + + ⛔ IT CATCHES ONLY THE MISMATCH AND LETS THE OTHER `RuntimeError` THROUGH, because the two mean + opposite actions and every caller already has a policy for the second one: + + * `DatastoreMismatch` — this process has ANOTHER tenant's file open. It never fixes itself by + waiting; the deployment is mis-pinned. Returned as a SENTENCE so each door can choose its + own status (the row doors raise, the status door reports — R6's second sentence). + * a plain `RuntimeError` — the mirror is completing its first sync, and retrying works. It + PROPAGATES, so `odoo_table_rows` keeps answering `503 store_not_ready` and the status door + keeps degrading to "no size half", exactly as each did before this guard existed. + + ⚠ Collapsing them was the tempting simplification and it is the defect: `DatastoreMismatch` + SUBCLASSES `RuntimeError`, so a single `except RuntimeError` here would render every + cross-tenant refusal as a retry banner. Catching the subclass first is a correctness rule. + """ + rtm = _rt() + try: + return rtm.mirror_cursor(session.runtime), None + except rtm.DatastoreMismatch as e: + return None, str(e) + + +@router.post("/odoo-tables/refresh") +def refresh_odoo_tables(session: Session = Depends(require_session)): + """Rebuild all four locked Odoo databases from the tenant's mirror — customers, products, + invoices and orders (it was invoices + customers until 2026-08-09). + + Idempotent: row ids are Odoo ids, so a re-run updates in place. Returns what MOVED, because + "refreshed" with no counts is indistinguishable from a no-op over an empty mirror. + """ + # ⛔ `deps.err()` RETURNS an HTTPException, it does not raise one — so it must be `raise + # err(...)`. `return err(...)` serialises the exception object with a **200**, which is the + # wave-23 shape exactly: a finished route, green everything, wrong on the wire. The house + # convention is 243 `raise err` against 6 strays; this file uses `raise`. + rel = _rel() + # ⛔⛔ THIS LINE WAS `session.is_admin` AND IT IS WHY D-107 LOOKED LIKE A STORE PROBLEM FOR A + # DAY. `Session` is a plain dataclass with ONE admin accessor, the `.admin` property; there is + # no `is_admin` and no `__getattr__`, so the attribute lookup raised `AttributeError` — SEVEN + # LINES ABOVE the `try:` below, for every caller, admin or not. That is the whole explanation + # for the measured signature: a bare `500` carrying Starlette's default PLAIN-TEXT body + # instead of our JSON envelope, on a route whose own handler had just been taught to name the + # exception. The refresh never ran, never reached `plan()`, never touched the mirror. + # ⚠ The tell was in the diagnosis all along and was read as evidence about the STORE: "it is + # not a timeout, the same operation takes 16.8 s from a laptop". Correct, and the reason was + # that the request never got as far as doing any work at all. + # ⭐ It was the SOLE `session.is_admin` in the repo against ~40 `session.admin` — an + # unmounted-shaped defect that no gate could see, because `verify_api` pinned these two routes + # as MOUNTED and never CALLED them. That gate now calls them; see `verify_api`'s odoo-tables + # section and its negative control. + if not session.admin: + raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.") + if not rel.is_royal(session.tenant): + # A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty + # locked databases it can never fill would be worse than refusing. + raise err(400, "not_applicable", + "These databases are built from the Royal Imports Odoo mirror (R1).") + try: + out = rel.refresh(session.runtime, session.tenant, username=session.user) + # ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN. + # + # The spawn writes every bucket's rows straight into the tenant document by mutating it + # inside its own updater, so no guard in `core.user_tables` is on that path (measured, not + # assumed: `_ensure_table_inplace` never calls a function there). Stripping AFTER the write + # is what makes "a read-through grid stores no rows" true rather than intended — and the + # refresh is the only moment a stripped table can come back. + # ⚠ Reported in the response, because a silent 7 MB moving in or out of a tenant's + # document is exactly the kind of thing an operator should be able to see happening. + sync_read_through() + moved = _ut().strip_materialised(st=session.runtime) + return {"ok": True, **out, "unmaterialised": moved} + except rel.Refused as e: + # A refusal is the ANSWER, not a crash: the caller must see WHY nothing was written. + raise err(409, "refused", str(e)) + except RuntimeError as e: + # `datastore.ro_con()` raises this while the store is completing its first sync. Relaying + # it beats writing a partial table from a half-synced mirror. + raise err(503, "store_not_ready", str(e)) + except Exception as e: # noqa: BLE001 + # ⛔ AN UNEXPECTED FAILURE MUST STILL SAY WHAT IT WAS. Measured live 2026-08-09: this + # route answered a bare `500` and the operator had no way to learn why — the reason lived + # only in a container log nobody can reach from the product. A spawn that rewrites four + # locked databases is exactly the operation whose failure needs a sentence. + # ⚠ The TYPE is included deliberately: `BinderException: Referenced column "agent_id" not + # found` is a different action (wait for the column backfill) from a timeout or an auth + # failure, and "500" cannot tell them apart. + raise err(500, "refresh_failed", f"{type(e).__name__}: {e}") + + +@router.get("/odoo-tables/status") +def odoo_tables_status(session: Session = Depends(require_session)): + """Row counts + the newest `refreshed` stamp per table. + + ⚠ The stamp is the whole point while the resync wiring is outstanding: a locked collections + worklist that quietly stopped updating is a worklist that lies, so its age must be readable + without anybody running a refresh to find out. + """ + rel, out = _rel(), {} + import core.user_tables as user_tables + # ⭐ W30-T31 — THE ELIGIBILITY PASS RUNS HERE TOO, and the mirror cursor is taken best-effort: + # the freshness surface must answer on a box with no mirror (that is what it is FOR), so a + # store that is not ready costs the size half of the question, never the whole endpoint. + # ⭐⭐ W31-T45 / D-169 — THIS IS THE DOOR THAT WAS ALREADY LEAKING, and it leaked as COUNTS + # rather than as rows, which is why nothing caught it. There is no tenant gate above this + # line: `rel.is_royal(session.tenant)` appears once, as an `applicable` FIELD in the response + # 60 lines below. So a nurilab or gtmlab session took a cursor on tenant #0's `royal.duckdb`, + # `_source_for` read its columns and `_ds.window` counted its rows — real, plausible numbers + # belonging to another customer, returned 200 OK. + # ⛔ IT REPORTS RATHER THAN RAISING, deliberately, and the distinction is R6's second sentence: + # "which Odoo databases exist for me" is a legitimate question for any tenant and its honest + # answer here is *none* — a 409 would be refusing the question instead of the leak. So the + # mirror half is refused, `rowsFrom` says so per table, and the refusal rides the payload with + # its cause. A count that silently became `-1` would be the silent truncation R6 bans. + cur, mirror_refused = None, None + try: + cur, mirror_refused = _mirror_cur(session) + except Exception: # noqa: BLE001 + pass # mid-first-sync (or no mirror at all) — the document half answers + eligible, why_not = sync_read_through(cur) + # ⚠ ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)` + # while those were the only two; the 2026-08-09 widening added products and orders, and a + # hard-coded list here would have reported "everything is fine" over two databases it had + # stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count). + for _bucket, key, _label, _fields in rel.TABLES: + table = user_tables.get(key, st=session.runtime) or {} + rows = (table.get("rows") or {}) + stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()] + # ⛔⛔ A FRESHNESS SURFACE THAT LIES IS WORSE THAN NO FRESHNESS SURFACE, and this endpoint + # was one step from becoming one. Both numbers below are derived from `table["rows"]`, so + # the moment a grid stops materialising they would read `rows: 0, refreshed: ""` — an + # operator would see a database that looks EMPTY and STALE on a route whose own docstring + # says a worklist that quietly stopped updating must be legible without a refresh. So a + # read-through table is counted from the MIRROR and says where its count came from. + materialised = user_tables.materialises(key, st=session.runtime) + if not materialised: + try: + rows = {} + stamps = [] + spec = _source_for(cur, key) if cur is not None else None + if spec is not None: + from harness import datastore as _ds + frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") + else {"table": spec["table"]}) + n = _ds.window(select="1", where=spec.get("where") or "", + order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + else: + n = -1 # unknown, and never reported as zero + except Exception: # noqa: BLE001 + n = -1 + out[key] = { + "exists": bool(table), + "rows": (len(rows) if materialised else n), + # ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from + # nowhere must not be labelled with the source it did not come from. + "rowsFrom": ("document" if materialised + else ("refused" if mirror_refused else "mirror")), + "materialised": materialised, + # why this grid still keeps its rows in the tenant document, when it does not have to + "materialisedBecause": (why_not.get(key, "") if materialised else ""), + "refreshed": (max([s for s in stamps if s], default="") if materialised + else "live — read through the mirror"), + "locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE, + # W30/R7: whether this database is served THROUGH the mirror rather than from the copy + # above. Reported per table, because they convert one at a time and an operator + # reading `rows: 0` needs to know whether that means "empty" or "not stored here". + "readThrough": key in _sources(), + } + # ⭐ W30-T32 / R6's SECOND SENTENCE, APPLIED TO OUR OWN HALF-BUILT STATE. A read-through + # binding whose FIELD DECLARATION has not landed yet answers 404 on the rows route, and an + # operator would read that as "the grid does not exist" rather than as "half of it shipped". + # These are the two line grains: bound here, declared in `odoo_relational` by W30-T35. + # ⚠ It reports the KEYS, never a field list — inventing a contract here is exactly the second + # source of truth the ticket forbids. + pending = {k: {"readThrough": True, "declared": False, + "cause": "this connected grid has a read-through binding but no field " + "declaration in odoo_relational yet, so it cannot be opened", + "recommendation": "declare its fields + a TABLES row (W30-T35); the binding " + "and the window are already live"} + for k in _sources() if k not in out} + return {"ok": True, "applicable": rel.is_royal(session.tenant), "tables": out, + "bound_not_declared": pending, + # ⭐ W31-T45 / D-169 — R6's SECOND SENTENCE, ON A GUARD RATHER THAN ON A ROW CAP. A + # limit that genuinely cannot be removed must be REPORTED with its cause and a + # recommended fix; silence is the violation. `null` when the mirror answered. + "mirrorRefused": ( + {"cause": mirror_refused, + "effect": "sizes_unavailable", + "recommendation": "this deployment has another tenant's analytical store open; " + "sizes are withheld rather than read from it. Pin the process " + "to this tenant (AIOS_DUCKDB_PATH / datastore.use_path) to " + "restore them."} + if mirror_refused else None), + # ⭐ W31-T46 / D-160 — WHY THERE IS NO MIRROR, when there is none. The last time this + # was silent it read as a pinned tag and a store problem for days. `null` when the + # container has one; a sentence with a cause and a fix when it does not. + "mirrorSeed": _seed_state()} + + +def _seed_state(): + """`main.MIRROR_SEED` as a reportable block, or None when the mirror is present. + + ⚠ IMPORTED LAZILY AND FAIL-QUIET: `main` imports this router, so a module-level import here + would be a cycle, and a freshness surface must never 500 because a diagnostic was unavailable. + """ + try: + import main as _main + from harness import datastore as _ds + if _ds.DB_PATH.exists(): + return None + state = dict(_main.MIRROR_SEED) + if not state.get("cause"): + state["cause"] = "this container has no analytical mirror yet" + state["recommendation"] = ("the boot seed runs independently of AIOS_PREWARM; if this " + "persists, check HF_TOKEN on the deployment") + return state + except Exception: # noqa: BLE001 + return None + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ THE READ-THROUGH WINDOW — owner ruling R6 ("no cap on connected-source data") via R7. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# THE PROBLEM IT REPLACES, in the owner's numbers. `core.user_tables.MAX_ROWS = 60_000` bounds a +# `ut_*` table because ALL of a tenant's tables live in ONE JSON document that `Store.get` +# deep-copies per request (D-87: four Odoo databases = 20.7 MB, ~370 ms a copy). Orders at +# **32,826** is already 55% of that ceiling; order lines (**255,286**) are 4.3× over it and GL +# lines (**971,034**) 16× over, so those two could never be grids at all — `odoo_relational.plan` +# refuses them rather than truncating, which is the correct refusal of the wrong architecture. +# +# ⭐ "NO CAP" IS NOT A BIGGER NUMBER. Raising `MAX_ROWS` would make every request slower for every +# tenant, including the ones who never open an Odoo grid. The mirror ALREADY holds all of it +# uncapped — measured on this box: 963,783 GL lines counted in ~24 ms — so the fix is to stop +# copying and serve a WINDOW: the requested slice, plus a `SELECT count(*)` that tells the truth +# about the whole. +# +# ⛔⛔ `total` IS NEVER `len(rows)`. It comes from `datastore.window`'s own count statement over +# the SAME predicate. A window whose count is its own length is a fabricated aggregate wearing an +# authoritative face ([[no-unverifiable-aggregates]]). +# +# ⛔ PREDICATES PUSH DOWN, and this is the half a client cannot be trusted with. A filter chip +# evaluated over the 200 rows that happen to be in memory would report "40 matches" out of +# 971,034 — [[one-question-two-normalizers]] at scale. `harness/filter_sql.py` compiles OUR filter +# vocabulary to SQL and is the same evaluator the TS engine is held in step with, so the fold and +# the display answer one question. +# +# ⭐ THE SPEC BELOW IS A SECOND STATEMENT OF SOMETHING `odoo_relational`'s READER ALREADY SAYS, +# AND THAT IS THE REAL RISK HERE — not SQL injection. It is gated rather than trusted: +# `verify_scopes.section_read_through` runs the REAL reader over the REAL mirror as an ORACLE and +# asserts the windowed path agrees ROW FOR ROW and TOTAL FOR TOTAL. A binding that drifts from its +# reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as +# though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]). +# +# ⚠ ONE BUCKET IS BOUND HERE (orders). The other seven still serve from the materialised copy and +# say so on the wire (`readThrough: false`), because R6's second sentence — *"if there is lag or +# it can't be done, you need to explicitly tell me why and recommend a fix"* — makes an unconverted +# grid something to REPORT, not something to leave looking converted. Adding one is a spec row plus +# a green oracle check. + +#: `ut_*` key -> how to read that grid straight out of the mirror. +#: +#: `where`/`select` are SQL WE author (never a request value); every request value is bound through +#: `params` by `filter_sql`. `cols` maps a FIELD KEY (what `odoo_relational._fields()` declares, +#: and what a saved view's filters name) to `(sql expression, coercion)`. The coercion reproduces +#: the reader's own python cast, by CALLING the reader's helpers where one exists — `_as_date` and +#: `_in_scope` are imported, not re-implemented. +#: Keys a reader does not produce (link columns, `refreshed`) are absent here on purpose: they are +#: filled by the grid at render time exactly as they are on the materialised path. +GRID_SOURCES = {} + +#: ⭐⭐ W31-T49 / C4 — THE SOURCE PROVIDERS. Odoo builds its specs below; ANY OTHER connector adds +#: its own by registering a builder here, and `_sources()` folds them into the one registry every +#: door already reads. +#: +#: ⛔ THIS IS THE SEAM THAT KEEPS "ONE REGISTRY" TRUE WHILE THE FILE STAYS ODOO-NAMED. The +#: alternative — a second `META_GRID_SOURCES` consulted beside this one — would give the platform +#: two answers to *"how is this connected grid read?"*, and every consumer (`_source_for`, +#: `whole_pool`, `population`, `odoo_tables_status`, `routes_tables.scoped_pids`) would have to +#: learn both or silently serve one. That is [[one-question-two-normalizers]] on the read path. +#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first +#: read — the same lazy rule `_sources` already follows for `odoo_relational`. +_SOURCE_PROVIDERS = [] + + +def register_source_provider(fn): + """Add a `() -> {table_key: spec}` builder to the connected-grid registry. + + Additive and idempotent by identity, so a re-import cannot double-register. Returns the number + of providers now known — a caller that wants to assert its registration took has a number. + """ + if callable(fn) and fn not in _SOURCE_PROVIDERS: + _SOURCE_PROVIDERS.append(fn) + return len(_SOURCE_PROVIDERS) + + +def _sources(): + """Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them. + + ⛔⛔ ODOO GOES THROUGH THE SEAM TOO, AND THAT IS THE POINT OF W31-T49 RATHER THAN A FLOURISH. + The first version built Odoo's specs inline here and folded other providers in afterwards, + which quietly says "Odoo is the registry and everyone else is an addendum" — two mechanisms + for one question, with the second one exercised by nobody until a connector arrives. It also + left `register_source_provider` with no production caller at all, which `verify_reachability` + correctly reddened as a capability behind no door ([[artifact-with-no-importer]]). Registering + the incumbent through its own seam makes the seam load-bearing from the first request. + + ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and + importing that module at file-import time would drag the Odoo layer into every process that + mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read. + """ + if GRID_SOURCES: + return GRID_SOURCES + for build in list(_SOURCE_PROVIDERS): + try: + extra = build() or {} + except Exception: # noqa: BLE001 + # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must + # not go dark because a newer connector's module is unhappy — and vice versa. + continue + if extra: + GRID_SOURCES.update(extra) + _ut().register_connected(*extra) + return GRID_SOURCES + + +def _odoo_sources(): + """Odoo's read-through bindings — `{table_key: spec}`. Registered as a provider below. + + ⚠ IT RETURNS a dict rather than mutating the module global: `_sources` owns the merge, so a + provider that half-built its specs and raised cannot leave a partial registry behind. + """ + rel = _rel() + # ⭐ R6 / W30-T29 — TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops + # being a fact about them. `core` never imports up, so the declaration goes this way round. + # ⚠ ALL EIGHT, not just the read-through-bound ones — every row in these tables comes from + # Odoo, which is what R6 is about; being served from the stored copy today is our conversion + # state, not a property of the data. `_sources`' own `register_connected` covers only the keys + # a provider RETURNS, so this call is not redundant with it and must not be folded into it. + _ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES]) + GRID_SOURCES = {} # the LOCAL registry this builder fills and returns + _s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)), + (lambda v: float(v or 0.0))) + GRID_SOURCES[rel.ORDERS_KEY] = { + "table": "sale_order", + # ⛔ IMPORTED, NOT RETYPED. `_CONFIRMED` is the fixed wholesale scope; if it ever changes, + # this window changes with it and the oracle check proves it did. + "where": f"{rel._CONFIRMED} AND partner_id IS NOT NULL", + "id": "id", + "needs_excluded": True, # `wholesale_scope` is resolved against the excluded set + "cols": { + "order_no": ("name", _s), + "odoo_id": ("id", _i), + "customer": ("partner_name", _s), + rel.JOIN_KEY: ("partner_id", _i), + "order_date": ("date_order", rel._as_date), + "amount_untaxed": ("amount_untaxed", _n), + "team": ("team_name", _s), + "state": ("state", _s), + "invoice_status": ("invoice_status", _s), + # ⭐ THE SCOPE COLUMN BECOMES REAL SQL, which is the point. On the materialised path it + # is `_in_scope(pid, excluded)` — a python set test, and a filter on it therefore could + # not push down. Inlining the ids (ints, from our own query) makes it a column the + # mirror can filter and sort on, so the R6 "limit" it would otherwise have earned does + # not exist. `{excluded}` is substituted by `_source_for` below. + "wholesale_scope": ("CASE WHEN partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + } + + # ═════════════════════════════════════════════════════════════════════════════════════════ + # ⭐⭐ W30-T32 — THE TWO LINE GRAINS. These are the grids R6 exists for: they have never had a + # `ut_*` table and never can, at any cap. MEASURED on this box's mirror, warm: + # sale_order_line 254,189 in the confirmed scope (256,810 unscoped) — 63.9 MB as JSON + # account_move_line 963,783 — ~240 MB as JSON + # Against `MAX_ROWS = 60_000` that is 4.2x and 16x, and against the 32 MB per-table document + # budget it is 2x and 7.5x. Read THROUGH, both serve a page in 134–166 ms. + # + # ⛔ THE FIELD KEYS BELOW ARE HALF OF A CONTRACT AND `odoo_relational` OWNS THE OTHER HALF + # (W30-T35, session E). `cols` binds a field key to SQL; the field's label, type and order are + # DECLARED THERE, once. Until that declaration lands the route answers 404 for these two keys + # (`rel.TABLES` has no entry), and `odoo_tables_status` REPORTS them as bound-not-declared + # rather than leaving them invisible — R6's second sentence applied to our own conversion. + # ⚠ A key here with no declaration there is a silent NO-CELL (`rows_from_pool` projects + # strictly); a key there with no binding here is an INACTIVE filter leaf, which WIDENS. The + # two lists are checked against each other by `verify_scopes.section_line_grids`. + ol_key = getattr(rel, "ORDER_LINES_KEY", "ut_odoo_order_lines") + gl_key = getattr(rel, "GL_LINES_KEY", "ut_odoo_gl_lines") + _ut().register_connected(ol_key, gl_key) + # ⛔ THE SCOPE IS THE ORDER'S, AND THE LINE TABLE CANNOT ANSWER IT ALONE: `sale_order_line` + # carries no `state` (12 columns, measured), so the confirmed-order scope — and `order_date`, + # and the order NAME a person reads the grid by — only exist across the join. MEASURED, warm, + # best of two: the JOIN beats `order_id IN (SELECT id FROM sale_order WHERE …)` at both depths + # (166 / 483 ms against 237 / 565 ms at offset 0 / 200,000), so the shape is chosen on a + # number rather than on taste. + # ⚠ `_CONFIRMED` is IMPORTED and QUALIFIED, never retyped — it opens with the bare column + # `state`, which `sale_order_line` does not have, so the prefix is what keeps it unambiguous + # if that table ever gains one. `section_line_grids` counts the same population a second way + # (a subquery, not a join) and the two must agree, which is what catches a mis-qualification. + GRID_SOURCES[ol_key] = { + "from_sql": "(sale_order_line sol JOIN sale_order so ON so.id = sol.order_id)", + "tables": {"sol": "sale_order_line", "so": "sale_order"}, + "where": f"so.{rel._CONFIRMED}", + "id": "sol.id", + "needs_excluded": True, + "cols": { + "odoo_id": ("sol.id", _i), + "order_no": ("so.name", _s), + "order_id": ("sol.order_id", _i), + "customer": ("sol.order_partner_name", _s), + rel.JOIN_KEY: ("sol.order_partner_id", _i), + "product": ("sol.product_name", _s), + rel.PRODUCT_JOIN_KEY: ("sol.product_id", _i), + "qty": ("sol.product_uom_qty", _n), + "price_subtotal": ("sol.price_subtotal", _n), + "margin": ("sol.margin", _n), + "purchase_price": ("sol.purchase_price", _n), + "order_date": ("so.date_order", rel._as_date), + "state": ("so.state", _s), + "wholesale_scope": ( + "CASE WHEN sol.order_partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + } + # ⚠ UNSCOPED ON PURPOSE, and it is a decision rather than an omission: every other grid here + # carries a fixed scope, but a GENERAL LEDGER whose draft and cancelled entries are invisible + # is a ledger that cannot be reconciled. `parent_state` rides as a column so a person filters + # in SQL over all 963,783 rows instead of us choosing for them. (Posted-only is 944,846.) + # The join to `account_account` is what makes `account_code` — the key `ut_odoo_accounts` is + # linked on — available at all; the mirror flattens `account_id`/`account_name` onto the line + # but not the CODE, and 192 accounts hash-join for free. + GRID_SOURCES[gl_key] = { + "from_sql": ("(account_move_line aml LEFT JOIN account_account aa " + "ON aa.id = aml.account_id)"), + "tables": {"aml": "account_move_line", "aa": "account_account"}, + "where": "", + "id": "aml.id", + "needs_excluded": True, + "cols": { + "odoo_id": ("aml.id", _i), + "entry": ("aml.move_name", _s), + "move_id": ("aml.move_id", _i), + "account": ("aml.account_name", _s), + rel.ACCOUNT_JOIN_KEY: ("aa.code", _s), + "customer": ("aml.partner_name", _s), + rel.JOIN_KEY: ("aml.partner_id", _i), + "date": ("aml.date", rel._as_date), + "debit": ("aml.debit", _n), + "credit": ("aml.credit", _n), + "balance": ("aml.balance", _n), + "line_type": ("aml.display_type", _s), + "move_type": ("aml.move_type", _s), + "parent_state": ("aml.parent_state", _s), + "wholesale_scope": ( + "CASE WHEN aml.partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + } + # ⭐ W30-T31 — THE GL ACCOUNT REGISTRY, BOUND BECAUSE IT IS THE ONE GRID THAT CAN ACTUALLY + # STOP MATERIALISING TODAY. 192 rows, nothing folds it, nothing links at it, and it fits + # inside one window — the three conditions `_unmaterialisable` checks. It is small, and that + # is the point: it is the first shipped database whose rows are NOT in the tenant document, + # so the stratum is proven on a real table instead of on a mechanism with no subject. + # ⚠ `account_fields()` declares six columns and `read_accounts` is one `cur.execute` over + # `account_account`; `section_read_through`'s differential oracle holds this binding to it. + # ⚠ `is_expense` REPRODUCES `read_accounts`' predicate IN SQL rather than inventing one, and + # that predicate is itself a copy of the semantic layer's `gl_lines` scope. Three statements of + # one rule is two too many, but the reader's own comment explains why it is copied rather than + # imported, and `section_read_through`'s differential oracle is what keeps this one honest. + GRID_SOURCES[rel.ACCOUNTS_KEY] = { + "table": "account_account", + "where": "", + "id": "id", + "cols": { + rel.ACCOUNT_JOIN_KEY: ("code", _s), + "account_name": ("name", _s), + "odoo_id": ("id", _i), + "account_type": ("account_type", _s), + "is_expense": ("CASE WHEN account_type IN ('expense','expense_depreciation') " + "THEN '1' ELSE '' END", _s), + }, + } + return GRID_SOURCES + + +#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers +#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`, +#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid. +register_source_provider(_odoo_sources) + + +def sync_read_through(cur=None): + """Register every grid that may stop storing rows, and REPORT why the rest may not. + + Returns `({key: eligible}, {key: reason})`. Idempotent, cheap, and safe to call from any door: + registration is additive and `strip_materialised` is a no-op once a table is empty. + + ⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility + question needs BOTH halves — `odoo_relational`'s field declarations (for the fold matrix) and + the mirror (for the size) — and `core` may import neither. + """ + eligible, reasons = _unmaterialisable(cur) + keys = [k for k, ok in eligible.items() if ok] + if keys: + _ut().register_read_through(*keys) + return eligible, reasons + + +#: ⚠ A mirror can be `ready()` and still lack a column (`ready()` reads entity PHASES; column +#: backfills checkpoint separately) — the gap that already cost a live 500. Every projected +#: expression is checked against the real column list and degraded to a literal, exactly as +#: `odoo_relational._col` does for the reader, so a fresh Space serves a blank cell rather than a +#: DuckDB Binder error. +#: +#: ⛔ W30-T32 — IT TAKES AN ALIAS MAP NOW, AND WITHOUT THAT THE GUARD WAS ABOUT TO GO BLIND. The +#: line-grain grids project `sol.price_subtotal` / `aml.parent_state`, and a dotted string is not +#: `isalnum()`, so the old single-table version returned EVERY qualified expression unchecked — +#: the same "expression, nothing to check" branch that correctly skips a CASE. Both of those +#: columns are 2026-07-28 backfills that a mirror can genuinely be missing, so the blind spot +#: would have surfaced as a bare DuckDB Binder error on a fresh Space, which is precisely the +#: failure this helper exists to prevent ([[gate-answers-the-wrong-question]]). +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W30-T31 / D-87 — WHICH CONNECTED GRIDS MAY STOP MATERIALISING, AND WHY MOST MAY NOT YET. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The prize is real: all of a tenant's `ut_*` tables live in ONE document that `Store.get` +# deep-copies on EVERY call, hit or miss, and the four original Odoo grids are 20.7 MB / ~370 ms +# of it. Every permission check in the app pays that. +# +# ⛔ AND YOU CANNOT SIMPLY DELETE THE ROWS, WHICH IS THE FINDING THIS FUNCTION ENCODES. Measured +# mechanically across the eight field declarations, not by eye: +# * `ut_odoo_invoices` and `ut_odoo_orders` — 16.2 of those 20.7 MB — are folded by SIX link +# rollups on `ut_odoo_customers` (`ar_outstanding`, `open_invoices`, `oldest_due`, +# `invoiced_all_time`, `order_count`, `last_order`). `automation_engine.compute_relation_cells` +# answers those by reading the RAW document, so with the rows gone it writes zeros — silently. +# * every table with a `link` pointing AT it (agents, vendors, bills, customers) has its link +# CELLS materialised the same way, from the target's stored rows. +# * `ut_odoo_customers` and `ut_odoo_products` carry SOURCE rollups, and `rollup_sql.compute` +# writes those cells INTO their own stored rows: no rows, no cells. +# * and while the grid client still asks for a whole table (F's W30-T42 is what changes this), +# a population larger than one window could only be served by TRUNCATING it — which R6 +# forbids more strongly than it forbids a cap. +# +# ⭐ SO THE PREDICATE IS DERIVED FROM THE DECLARATIONS RATHER THAN LISTED. The day a lane converts +# those six link rollups to SOURCE rollups (`orders_ytd` on that same table is the precedent), or +# the day the client pages, the affected grids become eligible here with NO code change — and +# until then each one's reason is reported per table on the status door, which is R6's second +# sentence applied to our own conversion state. +def _fold_reasons(rel): + """`{table_key: "why its stored rows are still read by something else"}`. + + Read out of the FIELD DECLARATIONS themselves — one pass over `rel.TABLES`. A table absent + from this map is folded by nothing. + """ + reasons = {} + + def _add(key, why): + reasons.setdefault(str(key), []).append(why) + + for _bucket, key, _label, mk in rel.TABLES: + try: + fields = mk() + except Exception: # noqa: BLE001 + continue + by_key = {f.get("key"): f for f in fields if isinstance(f, dict)} + short = str(key).replace("ut_odoo_", "") + for f in fields: + if not isinstance(f, dict): + continue + if f.get("type") == "link" and (f.get("link") or {}).get("table"): + _add(f["link"]["table"], f"{short}.{f['key']} is a link whose cells are built " + f"from these rows") + if f.get("type") != "rollup": + continue + bag = f.get("rollup") or {} + if isinstance(bag.get("source"), dict): + _add(key, f"{short}.{f['key']} is a source rollup and its cells are written " + f"into these rows") + continue + tgt = ((by_key.get(str(bag.get("link") or "")) or {}).get("link") or {}).get("table") + if tgt: + _add(tgt, f"{short}.{f['key']} folds these rows") + return {k: "; ".join(v) for k, v in reasons.items()} + + +def _unmaterialisable(cur=None): + """`({key: eligible}, {key: reason})` — who may stop storing rows, and why the rest may not. + + ⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in + the first cut of this function. A table too big for the tenant document has NO materialised + option at all, so making it ineligible would have handed `odoo_relational.plan` a `row_limit` + of None and invited it to build 963,783 python dicts. The window ceiling is a different, much + softer thing: it only limits what the whole-table CLIENT door can serve today. + + rows > MAX_ROWS -> read-through REQUIRED (the document cannot hold it) + else if something folds it -> stays materialised (a fold over no rows writes ZEROS) + else if rows > WINDOW_MAX -> stays materialised until the client pages (W30-T42) + else -> eligible + + ⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never + part of the materialised spawn is eligible — a table is never freed by a question we skipped. + """ + rel = _rel() + folds, eligible, reasons = _fold_reasons(rel), {}, {} + # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT. + # W30-T35 declared the two line grains, which have no python row builder anywhere — their + # absence from `_READERS` IS that statement — so a table the spawn could never materialise + # started reading as spawned, and on a mirror-less deployment fell through to "its size could + # not be read" and came back INELIGIBLE. That inverts this function's own first law (size + # forces read-through; it never blocks it). Ask the question through the reader map, which is + # what actually decides whether a row could ever have been built. + spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})} + from harness import datastore + ut = _ut() + for key, spec in _sources().items(): + total = None + if cur is not None: + try: + frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") + else {"table": spec["table"]}) + total = datastore.window(select="1", where=spec.get("where") or "", + order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + except Exception: # noqa: BLE001 + total = None + if total is not None and total > ut.MAX_ROWS: + eligible[key] = True # no other home exists; this is not a choice + continue + if total is None and key not in spawned: + eligible[key] = True # never materialised, so the mirror is its home + continue + why = [] + if total is None: + why.append("its size could not be read from the mirror on this deployment") + if key in folds: + why.append(folds[key]) + if total is not None and total > datastore.WINDOW_MAX: + why.append(f"its {total:,} rows exceed the {datastore.WINDOW_MAX:,}-row window and " + f"the grid client still asks for whole tables, so un-materialising it " + f"today could only truncate") + eligible[key] = not why + if why: + reasons[key] = "; ".join(why) + return eligible, reasons + + +def _degrade(expr, have_by_alias, default_alias=""): + bare = expr.strip() + alias, _, col = bare.partition(".") + if col and alias.replace("_", "").isalnum() and col.replace("_", "").isalnum(): + have = have_by_alias.get(alias) + if have is None: + return expr # an alias we do not own — leave it to the caller's SQL + return expr if col.lower() in have else "NULL" + if not bare.replace("_", "").isalnum(): + return expr # an expression, not a bare column — nothing to check + # ⚠ A BARE COLUMN IN A JOINED SPEC RESOLVES THE WAY SQL RESOLVES IT — against every table in + # the FROM, not against nothing. Checking it against `have_by_alias[""]`, which a joined spec + # does not have, would degrade every such column to NULL: a silent blank cell, which is the + # failure this helper exists to avoid rather than to cause. + have = have_by_alias.get(default_alias) + if have is None: + have = set().union(*have_by_alias.values()) if have_by_alias else set() + return expr if bare.lower() in have else "NULL" + + +def _source_for(cur, table_key): + """The resolved spec for one connected grid, or None when this grid is not read-through yet.""" + spec = _sources().get(str(table_key or "")) + if not spec: + return None + from harness import datastore + # ⚠ `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table + # spec declares none and its columns are bare, so it degrades against `table` as before. + aliases = dict(spec.get("tables") or {}) + if spec.get("table"): + aliases.setdefault("", spec["table"]) + have_by_alias = {} + for alias, tname in aliases.items(): + cols = datastore.columns_of(tname, cur=cur) + if not cols: + return None # the mirror has no such table on this deployment + have_by_alias[alias] = cols + excluded = "" + if spec.get("needs_excluded"): + ids = sorted(int(p) for p in _rel().excluded_ids(cur)) + # `-1` keeps the IN-list non-empty and matches no Odoo id, so the SQL shape is constant + # whether or not this tenant excludes a channel. + excluded = ", ".join(str(i) for i in ids) or "-1" + cols = {} + for key, (expr, cast) in spec["cols"].items(): + cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr, + have_by_alias), cast) + return {**spec, "cols": cols} + + +#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in +#: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of +#: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of +#: the person BEFORE the wait rather than after it. +_DEEP_PAGE = 100_000 + + +#: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our +#: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid +#: cannot disagree about whether a column is text or a number. +_FILTER_TYPE = {"select": "status", "checkbox": "text"} + + +def _filter_columns(spec, fields): + """`{colId: {sql, type, aggregate}}` — what `filter_sql` needs to compile a predicate. + + Only columns with a real mirror expression are offered. An omitted column is UNKNOWN to the + compiler, which skips it — and that is the one behaviour that must be REPORTED rather than + accepted, because an ignored condition WIDENS (the tri-state engine's inactive-leaf rule). + `_unpushable` below turns every such skip into an R6 sentence. + """ + by_key = {f["key"]: f for f in fields} + out = {} + for key, (expr, _cast) in spec["cols"].items(): + ftype = str((by_key.get(key) or {}).get("type") or "text") + out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False} + return out + + +def _leaf_cols(nodes): + """Every `colId` a filter tree names, at any depth.""" + seen = set() + for n in (nodes or []): + if not isinstance(n, dict): + continue + if n.get("children") is not None: + seen |= _leaf_cols(n.get("children")) + elif n.get("colId"): + seen.add(str(n["colId"])) + rhs = n.get("rhs") + if isinstance(rhs, dict) and rhs.get("colId"): + seen.add(str(rhs["colId"])) + return seen + + +def _json_arg(raw, what): + """Decode a JSON query argument, or refuse. ⛔ NEVER degrade to "no filter": a filter that + silently fails to parse WIDENS the answer, and the caller sees a plausible bigger number.""" + if raw in (None, ""): + return None + try: + val = json.loads(raw) + except Exception: # noqa: BLE001 + raise err(400, "bad_argument", f"{what} must be JSON") + return val + + +@router.get("/odoo-tables/{table_key}/rows") +def odoo_table_rows(table_key: str, + offset: int = Query(default=0, ge=0), + limit: int = Query(default=0), + filters: str = Query(default=None), + filterConj: str = Query(default="and"), + sorts: str = Query(default=None), + search: str = Query(default=None), + session: Session = Depends(require_session)): + """ONE WINDOW over a connected grid, read straight from the mirror (contract C2). + + `{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` — + `rows` is the requested slice and `total` is the count of everything the CURRENT PREDICATE + matches, from its own `SELECT count(*)`. `rows.length < total` is the normal case. + + ⛔ `total` is never `len(rows)`; `totalUnfiltered` is the population with the predicate + dropped, so a client can render "N of M" without inventing either number. + ⛔ The filter, the sort and the search all resolve in SQL against the whole table. Anything + that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in + `limits` with its cause and a recommendation — R6's second sentence — and never silently + dropped, because an ignored condition widens. + """ + import aios_grid + from harness import datastore + from routes_tables import _defn_or_refuse + + rel = _rel() + # THE WALL FIRST, and it is the SAME one the materialised path uses — 404 for a key that does + # not exist, 403 for one this session may not open. Reused rather than re-stated: a second + # idea of "may this session open this database" is a permission bug waiting to happen. + # + # ⭐⭐ W33-T02 / D-213 — `defs_only=True`, AND THIS ROUTE IS THE CLEANEST OPT-IN ON THE BOARD. + # `defn` is read EXACTLY ONCE below, for `recordMode`; the fields come from `rel.TABLES`'s + # `mk_fields()` and the rows from `datastore.window`, so nothing here has ever touched + # `defn["rows"]`. That is what makes it safe by inspection rather than by argument — and it is + # why the same change must NOT be swept across this file: `odoo_tables_status` a few hundred + # lines down reads `table["rows"]` and its per-row `refreshed` stamps for the MATERIALISED + # grids, and a projection there raises `KeyError` on every one of them. + # ⚠ `mirror_stamp`'s bare `except: return ""` would convert exactly that raise into a silently + # BLANK `refreshed` column rather than a red — which is why the boundary is drawn here, at the + # one function that provably needs no row, instead of at the file. + # W36-T21 ON THIS DOOR, AND `scope_applied=True` IS A PROMISE THE REST OF THIS FUNCTION KEEPS. + # `_defn_or_refuse` REFUSES 409 for a principal carrying a wall unless the caller declares it + # will apply one, and it defaults to False precisely so a door added without thinking is + # refused. This door was that door: it is the ONE route the client fetches rows from for a + # read-through database (`apiBridge.ts` builds `odoo-tables//rows`), and every one of + # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a + # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows + # used to be. So the wall is APPLIED here, in SQL, three paragraphs down. + defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + fields = None + for _bucket, key, _label, mk_fields in rel.TABLES: + if key == table_key: + fields = mk_fields() + break + if fields is None: + raise err(404, "not_connected", "that database is not a connected Odoo grid") + + # ⭐⭐ W31-T45 / D-169 — THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot. + # That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process + # has open the one this TENANT's rows live in". A definition wall is satisfied the moment a + # tenant's own document declares a connected table — which is exactly what R2's Meta Ads spawn + # gives GTM Lab — and it would then serve that session a window over whatever DuckDB file the + # process happens to be pinned to. Rows, not counts. So this door RAISES. + # ⛔ 409, NOT 503, and the two were one line from being confused: `DatastoreMismatch` subclasses + # `RuntimeError`, and the handler directly below turns a `RuntimeError` into + # `503 store_not_ready` — "the store is completing its first sync, retry in a few minutes". A + # mis-pinned process never becomes un-mis-pinned by waiting, so relaying it as a retry would + # turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the + # try, not inside it. + try: + cur, mismatch = _mirror_cur(session) + except RuntimeError as e: + # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the + # mismatch subclass, which is what makes this broad clause safe to keep here. + raise err(503, "store_not_ready", str(e)) + if mismatch: + raise err(409, "cross_tenant_store", mismatch) + + spec = _source_for(cur, table_key) + if spec is None: + raise err(409, "not_read_through", + "this connected database is still served from its stored copy; it has no " + "read-through binding on this deployment yet") + + cols = _filter_columns(spec, fields) + + # ── THE PERMANENT WALL (W36-T21 / R6), COMPILED FIRST AND FAIL-CLOSED ──────────────────── + # An administrator's row filter is a `FilterTree` in the SAME vocabulary the client sends, so + # it compiles through the SAME compiler and ANDs into the SAME `where`. Pushed down rather + # than applied to the window, because a wall applied AFTER `LIMIT/OFFSET` returns a short page + # and a `total` that counts rows the reader may not have - two wrong numbers on screen. + # + # A WALL THAT CANNOT BE COMPILED REFUSES. `perm_scope.permits()` denies on a predicate it + # cannot answer, and the SQL door must agree: a condition dropped here does not narrow, it + # WIDENS, and it would do so silently under a `limits` note nobody reads as a permission + # failure. The three ways it can fail (a column with no mirror expression, an op with no SQL + # form, an aggregate) each answer 409 naming the cause, which is R6's second sentence. + # + # AND THE FIELD WALL RIDES WITH IT. Hidden columns leave `cols` before the USER's predicate, + # search and sort are compiled, so a scoped reader cannot infer a hidden column's values by + # filtering on it - a condition on one is reported unanswerable instead of evaluated. The + # wall's OWN predicate compiles against the full contract on purpose, exactly as + # `perm_scope._scoped` evaluates it against the unstripped fields. + import core.perm_scope as perm_scope + wall_sql, wall_params = None, [] + _hidden = perm_scope.hidden_keys(session.user, table_key, fields) + # `row_scope_applies` RATHER THAN AN ADMIN TEST SPELLED OUT HERE: it is this repo's ONE + # statement of when the row wall bites (False for an admin, False for a record with no + # declared filter), and a second spelling of it beside the first is how two of them come + # apart. It also keeps the admin break-glass identical on this door and on every other. + _wall_tree = ((perm_scope.entry(session.user, table_key) or {}).get('filter') + if perm_scope.row_scope_applies(session.user, table_key) else None) + # READ THROUGH `tree_parts`, WHICH IS THE ONE READER OF THIS SHAPE. A stored wall is + # `{conj, nodes}` (C-PERM amendment 2) and BOTH consumers below take a BARE NODE LIST plus a + # separate conj - `_leaf_cols` iterates its argument, and `compile_filter_tree(nodes, conj=)` + # wraps it. Handing either the dict is silently empty rather than an error: `_leaf_cols` + # iterates the dict's KEYS, finds no dict among them, and answers 'this wall names no + # columns' - so an unanswerable wall would have passed the check below and then compiled to + # nothing, i.e. no wall at all, on the one path where that means serving the whole database. + _wall_nodes, _wall_conj = _fe().tree_parts(_wall_tree) + if _wall_nodes: + _unknown = sorted(_leaf_cols(_wall_nodes) - set(cols)) + if _unknown: + raise err(409, "scope_not_applicable", + "an administrator restricted your view of this database using " + + ", ".join(_unknown) + + ", and that column cannot be read on this database's live connection, so " + "the restriction cannot be applied here. Ask an administrator to restrict " + "you on a column this database carries") + try: + _wall = _fs().compile_filter_tree(_wall_nodes, conj=_wall_conj, columns=cols, + today=_today()) + except ValueError as e: + raise err(409, "scope_not_applicable", + "an administrator restricted your view of this database with a condition " + "this database's live connection cannot evaluate (" + + str(e) + "), so it cannot be applied here rather than ignored") + if _wall is not None: + if _wall.uses_aggregate: + raise err(409, "scope_not_applicable", + "an administrator restricted your view of this database with a " + "condition on an aggregate column, which this database's live " + "connection cannot evaluate one row at a time") + wall_sql, wall_params = _wall.sql, list(_wall.params) + if _hidden: + cols = {k: v for k, v in cols.items() if k not in _hidden} + + # THE BASELINE EVERY COUNT ON THIS RESPONSE IS TAKEN AGAINST. `totalUnfiltered` means "the + # population with YOUR conditions dropped" and it must never mean "with the WALL dropped" - + # that number would tell a scoped reader exactly how many rows they are not allowed to see. + base_where = (f'({spec["where"]}) AND {wall_sql}' + if (spec["where"] and wall_sql) else (wall_sql or spec["where"])) + limits, tree = [], _json_arg(filters, "filters") + sort_spec = _json_arg(sorts, "sorts") or [] + + # R6's SECOND SENTENCE, ON THE WALL ITSELF. The precision note further down covers the + # conditions a USER sends; the same arithmetic governs the one an ADMINISTRATOR stored, and + # a scoped reader has no other way to learn that the boundary they are held to is evaluated + # at whole units while the cells beside it carry cents. Reported, never silently enforced. + _wall_numeric = sorted(k for k in (_leaf_cols(_wall_nodes) & set(cols)) + if cols[k]["type"] in ("currency", "int", "pct")) + if wall_sql and _wall_numeric: + limits.append({ + "subject": ", ".join(_wall_numeric), "effect": "precision", + "cause": "the restriction an administrator set on your view of this database is " + "evaluated in SQL at whole-unit precision, while these cells carry their " + "exact value, so a row within half a unit of the boundary can fall on " + "either side of it", + "recommendation": "ask an administrator to set the restriction on a whole number, " + "or on a column that is not an amount"}) + + # ── the predicate, pushed down ──────────────────────────────────────────────────────────── + named = _leaf_cols(tree) + missing = sorted(named - set(cols)) + if missing: + limits.append({ + "subject": ", ".join(missing), "effect": "filter_ignored", + "cause": "these columns have no expression in the mirror (a link, a rollup, or a " + "column this deployment's mirror has not backfilled), so a condition on " + "them cannot be answered in SQL", + "recommendation": "filter on the id column the link is built from, or open the " + "linked database directly"}) + # ⭐ R6, MEASURED, AND IT IS THE KIND OF LIMIT THE RULING EXISTS FOR — a difference in the + # ANSWER, not in the speed. + # + # `filter_sql._value_sql` compiles every numeric comparison as `round_even(x, 0)`, and says + # why: *"reproduces `aios_grid._round` … The grid displays rounded values; filters must agree + # with what is on screen."* That is true of `source: "odoo"` columns, which `rows_from_pool` + # rounds. It is FALSE here — these columns are `source: "overlay"` (a storage choice, not a + # display one) so `rows_from_pool` passes the exact value through, and the TS engine + # (`useVisibleRows.toNum`) does not round either. So the pushdown compares at whole units while + # the cell beside it carries cents. + # + # MEASURED on the live orders mirror (32,700 rows, 47.9% with a non-integer amount): + # > 1000 python 4,476 vs SQL 4,473 (-3) + # > 173.6 python 24,711 vs SQL 24,726 (+15) + # > 500.25 python 11,028 vs SQL 11,026 (-2) + # Small, and NOT nothing. Rounding the wire to match would have been the other fix and it was + # rejected on measurement: it changes 96 of every 200 money cells (173.55 -> 174) to remove a + # 0.05% counting difference — a visible product regression traded for an invisible one. + # ⛔ SO IT IS REPORTED INSTEAD. Booked for the owner of `harness/filter_sql.py`, which is not + # this fence; see mailbox/D.md. + numeric = sorted(k for k in (named & set(cols)) + if cols[k]["type"] in ("currency", "int", "pct")) + if numeric: + limits.append({ + "subject": ", ".join(numeric), "effect": "precision", + "cause": "a number condition is evaluated in SQL at whole-unit precision " + "(`filter_sql` rounds to match the grids whose values the server rounds), " + "while these cells carry their exact value — so a row within half a unit of " + "the threshold can fall on the other side of it", + "recommendation": "compare against a whole number, or use a range that does not sit " + "on a fractional boundary"}) + + where, params = base_where, list(wall_params) + try: + pred = _fs().compile_filter_tree( + tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols, + today=_today()) + except ValueError as e: + # ⛔ A REFUSAL IS AN ANSWER, NOT A CRASH — and it must not become "no filter". Ranking ops + # ("top 10") have no SQL form in this compiler; saying so beats returning every row. + raise err(400, "filter_unsupported", str(e)) + if pred is not None: + if pred.uses_aggregate: + raise err(400, "filter_unsupported", + "a condition on an aggregate column belongs in HAVING, and that path is " + "deliberately not built for windowed grids") + where = f"({where}) AND {pred.sql}" if where else pred.sql + params.extend(pred.params) + if str(search or "").strip(): + got = _fs().compile_search(search.strip(), cols) + if got is not None: + where = f"({where}) AND {got.sql}" if where else got.sql + params.extend(got.params) + + # ── the order, made TOTAL ───────────────────────────────────────────────────────────────── + # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one + # row on two pages and drop another entirely — a duplicate the user sees with no error + # anywhere. `compile_order_by`'s own docstring says so. + order = _fs().compile_order_by(sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"] + + select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) + # ⚠ EXACTLY ONE of `table`/`from_sql` — `window` raises if both or neither arrive, so the + # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM. + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', + where=where, params=tuple(params), order_by=order, + offset=offset, limit=(limit or None), cur=cur, **frm) + if win["clamped"]: + limits.append({ + "subject": "limit", "effect": "window_clamped", + "cause": f"one response carries at most {datastore.WINDOW_MAX} rows so a single " + f"request cannot exhaust memory for every other tenant on the process", + "recommendation": "page with `offset`; `total` already reports the whole population, " + "and no row is unreachable"}) + # ⭐ R6's SECOND SENTENCE, ON THE ONE LIMIT THAT SURVIVES THE CONVERSION. Removing the row cap + # does not make every row equally cheap: `LIMIT/OFFSET` WALKS the offset, so the deeper the + # page the longer the scan. MEASURED warm on 963,783 GL lines — offset 0: 134 ms · 100,000: + # 71 ms · 900,000: **1,450 ms**; sorted by date rather than by id, offset 500,000: 1,645 ms. + # It is a real cost, it is nobody's mistake, and the owner asked to be told rather than to + # discover it: say so with the fix, which is a CURSOR the client has to send. + if offset >= _DEEP_PAGE: + limits.append({ + "subject": "offset", "effect": "slow", + "cause": f"a page {offset:,} rows deep is reached by walking every row before it " + f"(SQL OFFSET has no other meaning), which costs about a second past " + f"half a million rows", + "recommendation": "jump with a filter or a sort instead of scrolling, or ask for " + "keyset paging (`id > `), which is O(page) at any depth " + "and needs the client to send the last row it holds"}) + + # ── the wire rows, through the SAME serialiser the materialised path uses ────────────────── + # ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the + # `refreshed` column identically instead of one of them leaving it blank. + rows_src = _pool_from_window(spec, win, + stamp=mirror_stamp(spec, cur, table_key, + session.runtime)) + overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src} + + # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID. + # + # A user-added column on a connected grid has nowhere per-user to live: there is no `ut_*` row + # to hang it on any more, and `table_store`'s per-user strata would make a SHARED view name a + # column other accounts do not have — an unknown column is an INACTIVE condition in the + # tri-state engine, which WIDENS. `core/shared_overlay.py` was built for exactly this and has + # had no product door since it shipped (W29-T62). + # + # ⛔ `cells(table_key, pids)` TAKES THE PIDS AND THERE IS NO "EVERYTHING" CALL — and here that + # is an asset rather than a chore: **the window IS the scoped row set**, already narrowed by + # the wall and the predicate, so the argument it demands is the list we just fetched. + # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already + # answered "may this session open this surface". + shared_defs = _so().fields(table_key, st=session.runtime) + if shared_defs: + fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()] + for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src], + st=session.runtime).items(): + overlays.setdefault(pid, {}).update(cells) + # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above + # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula + # in an overlay column that reads a hidden mirror column was outside its reach - and a formula + # carrying a hidden value is the leak `hidden_keys` exists to close, wearing a second door's + # name. Same wall, asked once the field list is whole. + _hidden = perm_scope.hidden_keys(session.user, table_key, fields) + if _hidden: + # BOTH WIRES, and the field list is narrowed BEFORE the rows are built so a hidden column + # is never assembled rather than assembled and then removed. `strip_row` runs anyway on + # the overlay-merged result, because a shared-overlay cell arrives by a different door + # than the mirror window and one narrowing cannot speak for both. + fields = perm_scope.visible_fields(fields, session.user, table_key) + rows_src = [perm_scope.strip_row(r, _hidden) for r in rows_src] + overlays = {pid: {k: v for k, v in cells.items() if k not in _hidden} + for pid, cells in overlays.items()} + rows = aios_grid.rows_from_pool(rows_src, fields, overlays) + + unfiltered = win["total"] if where == base_where else datastore.window( + select="1", where=base_where, params=tuple(wall_params), order_by=spec["id"], + limit=1, cur=cur, **frm)["total"] + + return {"ok": True, "fields": fields, "rows": rows, + # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`. + "total": win["total"], "totalUnfiltered": unfiltered, + "offset": win["offset"], "limit": win["limit"], "limits": limits, + "today": _today(), "identity": {"pid": "pid"}, + "scope": {"table": table_key, "readThrough": True}, + "recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)} + + +def mirror_stamp(spec, cur=None, table_key="", rt=None): + """⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date. + + ⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table + declares `refreshed` (`odoo_relational._refreshed_field`), and a MATERIALISED grid gets a + per-row stamp written by the spawn. A read-through grid has no stored row to carry one, so + `_pool_from_window` produced no `refreshed` key at all and `rows_from_pool` projected it as + `""` — a freshness column that used to say something and now silently says nothing, on the two + biggest grids in the product. Nothing goes red for that; it just quietly stops being true. + + ⭐ THE HONEST VALUE ALREADY EXISTS — the mirror keeps a per-entity sync stamp in `_sync_state`, + which is the same fact one level up: these rows are as fresh as the last sync of the table + they are read through. It is per-TABLE, not per-row, and that is not a downgrade — for a grid + that stores no rows, "when did this table last sync" IS the per-row answer. + + ⚠ READ ON THE CALLER'S CURSOR, never `datastore.status()`. That opens a SECOND connection to a + file DuckDB holds exclusively, so a freshness column would turn a working grid into an + IOException — a cosmetic fix taking out the feature it decorates. + ⚠ Date only (`YYYY-MM-DD`), because `refreshed` is declared `type: "date"` and the grid's date + renderer is what reads it; handing it a full timestamp renders the ISO string a user should + never see (the wave-26 `copyData` scar, one column over). + + ⛔⛔ THE TABLE COMES FROM THE **ID COLUMN'S ALIAS**, AND THE FIRST DRAFT GOT THIS WRONG IN THE + ONLY WAY THAT MATTERS. It read `spec["table"]` with a `tables[""]` fallback — and **neither of + the two grids D-155 is about has a bare `table` key**: `ut_odoo_order_lines` and + `ut_odoo_gl_lines` are join-shaped (`from_sql` + `tables: {"sol": …, "so": …}` / + `{"aml": …, "aa": …}`), so the lookup returned `""` and the fix would have shipped doing + nothing on exactly the two grids it was written for — green gate, unchanged product. It was + caught only because the gate's fixture used `ut_odoo_orders`, a MATERIALISED grid that never + had the defect [[gate-answers-the-wrong-question]]. + ⭐ The id column IS the grain: `sol.id` means this grid is one row per `sale_order_line`, and + the freshness of a joined lookup table (`sale_order`, `account_account`) is not this grid's + freshness. So the alias is read off `spec["id"]`, and a bare `id` degrades to `table` — which + is exactly the single-table case. + + ⛔ AND IT ANSWERS `""` FOR A **MATERIALISED** GRID, ON PURPOSE. Those eight tables store a + per-ROW `refreshed` written by the spawn, which is strictly better than one table-level date — + and this value arrives as an OVERLAY, so returning a stamp here would quietly overwrite eight + working grids' per-row stamps while fixing two blank ones. D-155's subject is the grid that has + NOWHERE to keep a row; a fix that also rewrites the grids that do is a different, unrequested + change [[reuse-and-delete-are-hypotheses]]. + """ + # the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot + # tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank), + # never a stamp that might overwrite a per-row one. + if table_key: + try: + if _ut().materialises(str(table_key), st=rt): + return "" + except Exception: # noqa: BLE001 + return "" + ident = str(spec.get("id") or "") + alias = ident.split(".", 1)[0] if "." in ident else "" + table = str((spec.get("tables") or {}).get(alias) or spec.get("table") + or (spec.get("tables") or {}).get("") or "") + if not table or cur is None: + return "" + try: + row = cur.execute("SELECT updated_at FROM _sync_state WHERE entity = ?", + [table]).fetchone() + except Exception: # noqa: BLE001 + return "" # no mirror bookkeeping ⇒ blank, exactly as before + return str((row or [""])[0] or "")[:10] + + +def _pool_from_window(spec, win, stamp=""): + """`[{pid, **cells}]` — THE one place a mirror window becomes product rows. + + ⛔ ONE BUILDER, TWO CALLERS, and that is deliberate rather than tidy: the windowed route and + the whole-table read-through below would otherwise each cast the same columns their own way, + and a cell that renders differently depending on which door served it is this repo's recorded + defect class ([[one-question-two-normalizers]]). + + ⚠ `stamp` (D-155) rides here for that same reason: both doors must produce the same + `refreshed` cell, and a caller that forgot it would give one door a freshness column and the + other a blank one. Empty when the caller cannot cheaply know — blank is what shipped, so the + degradation is the previous behaviour rather than a new wrong value. + """ + keys = list(spec["cols"]) + extra = {"refreshed": stamp} if stamp else {} + return [{"pid": int(r[0]), **extra, + **{k: spec["cols"][k][1](v) for k, v in zip(keys, r[1:])}} for r in win["rows"]] + + +class TooBigToMaterialise(Exception): + """A read-through table asked for WHOLE exceeds one window — refuse, never truncate.""" + + +def population(table_key, cur=None, rt=None): + """How many rows a read-through grid HAS, without fetching one — or None if it is not bound. + + ⭐⭐ W31 / B's ask (mailbox/B.md B-2). `routes_tables.scoped_pids` learned "this grid is too big + to list" the only way that existed: call `whole_pool()` and catch `TooBigToMaterialise`. That + pulls a full 5,000-row window out of DuckDB and throws it away on every `/workspace` for + `ut_odoo_gl_lines` and `ut_odoo_order_lines` — MEASURED by B at ~1,600 ms of a ~3,460 ms + in-proc envelope. This asks the SIZE instead: one `SELECT count(*)` over the same predicate. + + ⛔ IT ANSWERS A DIFFERENT QUESTION, NOT THE SAME ONE MORE CHEAPLY, and that distinction is what + keeps it safe. B's constraint is that the pid set must stay IDENTICAL to `scoped_pool`'s BY + CONSTRUCTION rather than by two queries that agree today ([[one-question-two-normalizers]]) — + so this returns a COUNT and never a pid. A caller uses it to decide whether to ask for pids at + all; when it does ask, the pids still come from the one `whole_pool` fetch they always did. + + ⚠ `None` means "no read-through binding on this deployment", which is NOT `0`. A caller that + collapses them reports an empty grid where it should report an unbound one — the exact + distinction `odoo_tables_status` already reports as `bound_not_declared`. + ⚠ `total` is the count over the spec's OWN `where`, i.e. the same population `whole_pool` + would return — the fixed scope is not dropped for being cheaper. + """ + from harness import datastore + if rt is not None: + rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry + cur = cur if cur is not None else datastore.ro_con() + spec = _source_for(cur, table_key) + if spec is None: + return None + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"], + limit=1, cur=cur, **frm)["total"] + + +def whole_pool(table_key, cur=None, rt=None): + """Every row of a read-through grid, in `scoped_pool`'s shape — or a refusal. + + ⚠ THIS EXISTS FOR THE CLIENT WE HAVE, NOT THE ONE WE WANT. The grid still fetches whole + tables (`GET /tables/{key}/rows`); paging is F's W30-T42. So a database whose rows have left + this tenant's document has to be servable to that client somehow, and the honest answer for a + small one is "read all of it from the mirror". `_unmaterialisable` only ever registers a table + that fits, so the refusal below is a guard against the population GROWING past the window + later — at which point R6 requires a sentence, not a quietly shorter grid. + + ⭐⭐ W31-T45 / D-169 — `rt` IS THE TENANT RUNTIME, AND IT IS OPTIONAL FOR A STATED REASON + RATHER THAN A LAZY ONE. This function serves ROWS, so it is the door where a cross-tenant read + would be worst — but its only production caller is `routes_tables.py::_read_through_rows`, + which is in another lane's fence THIS WAVE (B's W31-T20 is rewriting that exact path) and does + not thread a session down to here. Making `rt` required would break that caller on import; a + second predicate invented locally would be one question with two normalizers, a recorded + defect class here. So the guard fires when a caller passes the runtime it already holds, and + the ask to thread `session.runtime` through `_read_through_rows` is posted to B in + `mailbox/D.md`. ⛔ Until that lands this door is guarded only by `_defn_or_refuse` upstream — + stated here rather than left for a reader to discover, because an unguarded row door is + precisely what D-169 is about. + """ + from harness import datastore + if rt is not None: + cur = cur if cur is not None else _rt().mirror_cursor(rt) + rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent + cur = cur if cur is not None else datastore.ro_con() + spec = _source_for(cur, table_key) + if spec is None: + raise TooBigToMaterialise( + f"{table_key}: this database is served through the mirror but has no read-through " + f"binding on this deployment, so its rows cannot be read at all") + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) + win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=spec.get("where"), + order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm) + if win["total"] > len(win["rows"]): + # ⛔ REFUSE, NEVER TRIM. A short grid that says nothing is exactly the silent truncation + # R6's second sentence is about — and the caller turns this into a message with a cause. + raise TooBigToMaterialise( + f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row " + f"window, and this database no longer stores rows in the tenant document; it can " + f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)") + return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt)) + + +def _fe(): + # `filter_eval.tree_parts` is the ONE reader of a stored `FilterTree`'s `{conj, nodes}`. + # A sibling of `_fs()` rather than a top-level import for the same reason that one is: + # `harness` pulls in the grid stack and this module is on the request path. + from harness import filter_eval + return filter_eval + + +def _fs(): + from harness import filter_sql + return filter_sql + + +def _so(): + import core.shared_overlay as shared_overlay + return shared_overlay + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _today(): + import time + return time.strftime("%Y-%m-%d") diff --git a/api/routes_publish.py b/api/routes_publish.py index 3c9d85cd01091f4cb99ec7874e1647bf54ba60bd..b0a8bbebf8820fd13b3bc96e8cf175a815430a99 100644 --- a/api/routes_publish.py +++ b/api/routes_publish.py @@ -1,917 +1,917 @@ -"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). - -The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface -(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed -publicly or with a password that the user that share it can toggle."* - -R5, in five clauses, and every one of them is load-bearing: - * publish = a per-view secret token in a **server-only bucket**; - * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; - * an **unauthenticated** read-only route plus a `#/v/` client route; - * the publish surface **projects only the columns the view shows** — never the whole table; - * **creator-or-admin** may publish, and **revoke ROTATES** the token. - -⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door -has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal -for every resolution failure (so the route is not an oracle for which tokens are real), a -constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the -whole body before the handler's first line, and `content-length` is caller-supplied), a sliding -rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. - -⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE -door into a table; a publish token buys a READ projection of one view. Folding them into one index -would make a single leaked string ambiguous about which of the two it opens, and would make -`_resolve` return an object whose capability depends on a field rather than on which door was -knocked. Two buckets, two resolvers, one shape. - -⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and -they are not the same question (audit S-4): the grant registry drives *Shared with me*, -`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing -— an unauthenticated read of a projection, bound to a secret rather than to an account. A published -link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot -be revoked by removing a person, because there is no person. - - python verify_forms.py # this file's gate; section 6 onward -""" -import hashlib -import hmac -import os -import secrets -import time - -from fastapi import APIRouter, Depends, Request - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. -#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec -#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser -#: writes that spec on every autosave — so anything secret living there would be echoed back to the -#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. -TOKENS_KEY = "publish_tokens" -TOKEN_BYTES = 24 - -#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. -#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says -#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is -#: not a stronger string, it is that the secret is never stored, never logged and never echoed. -PW_ITERATIONS = 240_000 -PW_SALT_BYTES = 16 -MIN_PASSPHRASE = 6 -MAX_PASSPHRASE = 128 - -#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so -#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. -RATE_WINDOW_S = 60 -RATE_PER_WINDOW = 30 -MAX_BODY_BYTES = 16 * 1024 - -_HITS: dict = {} - -#: The view kinds that may be published, i.e. R5's "interface". -#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 -#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two -#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a -#: server list that silently drifts from the picker means a mode a user can create and cannot -#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row -#: set; publishing one would be publishing the table, which is exactly what R5's projection clause -#: exists to prevent. -#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. -#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a -#: row set the publisher already curated; a form's row set is other people's answers, gathered under -#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into -#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the -#: product, with no wall between the link and the data. -#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which -#: serves the BLANK form for submitting and never the stored rows. So this is not a capability -#: removed, it is a second door onto the same object that should never have existed beside the -#: first. Publishing a form to be filled in still works, at the URL that was always for it. -#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not -#: offer Publish on a form view, or the picker promises what this refuses. -PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") - - -def _refuse(): - """THE ONE REFUSAL, for every way a published link can fail to resolve. - - Wrong token, revoked link, deleted view, deleted table, a view that stopped being an - interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: - the difference between "no such link" and "that link was revoked" tells an enumerator which - tokens are real, and the difference between "no such link" and "wrong passphrase" tells them - which links are worth guessing at. - """ - return err(403, "bad_publish_token", "that link is not valid") - - -def _same(a: str, b: str) -> bool: - """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" - return hmac.compare_digest(str(a or ""), str(b or "")) - - -def _client_ip(request: Request) -> str: - """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. - - The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new - value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted - through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its - old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the - use it could not support. It is safe as a LOG field and as nothing else. - - The socket peer is the only thing here the caller cannot choose, so it is the key. - ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES - ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm - that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer - is that we cannot separate callers, not that we should throttle them together. - """ - return request.client.host if request.client else "?" - - -def _rate_ok(key: str, now: float) -> bool: - """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. - - A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the - whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire - protection is that guessing a 24-byte token is slow. - - ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer - survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at - all; an enumerator produces nothing but misses. Counting every request under one shared peer - would have denied service to everybody the moment one guesser showed up — trading a token - oracle for an outage is not a fix. - """ - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - _HITS[key] = seen - return len(seen) < RATE_PER_WINDOW - - -def _note_failure(key: str, now: float) -> None: - """Record one failed resolution against this peer, and keep the table bounded.""" - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - seen.append(now) - _HITS[key] = seen - if len(_HITS) > 4096: - for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: - _HITS.pop(k, None) - - -#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash -#: to check against. Its value is irrelevant; its COST is the point. -_DUMMY_SALT = b"\x00" * 16 - - -def _equalise_pw_cost(entry) -> None: - """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. - - MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in - ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording - ("that link is not valid", identical for both) was undone by the clock: anyone could sort real - tokens from fake ones by timing alone, then spend their guesses only on the real ones. The - docstring on `_refuse` describes exactly the leak the code then handed over for free. - - Called on every path that fails BEFORE a passphrase check would have happened, so the cheap - branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so - a link minted under a different iteration count stays indistinguishable too. - """ - iterations = PW_ITERATIONS - if isinstance(entry, dict): - try: - iterations = int(entry.get("iter") or PW_ITERATIONS) - except (TypeError, ValueError): - iterations = PW_ITERATIONS - _hash_pw("", _DUMMY_SALT, iterations) - - -def _public_base() -> str: - return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") - - -def _link_of(token: str) -> str: - return f"{_public_base()}/#/v/{token}" - - -def _tokens(rt) -> dict: - """This tenant's publish index, always a dict.""" - try: - found = rt.get(TOKENS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: - return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), - salt, int(iterations)).hex() - - -def _pw_ok(entry: dict, passphrase: str) -> bool: - """Constant-time verify against the stored digest. - - ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" - from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not - fall open. [[default-must-pass-its-own-guard]] - """ - stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") - if not stored or not salt_hex: - return False - try: - salt = bytes.fromhex(salt_hex) - except ValueError: - return False - got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) - return _same(got, stored) - - -def _find_view(rt, table_key: str, view_id: str): - """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. - - The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` - does, and the reason a view is addressable by id ALONE here: the id is unique within the table, - while the owner is what we are trying to discover. - """ - if not table_key or not view_id: - return None, "" - try: - bucket = rt.get(f"{table_key}_table_workspace") or {} - except Exception: # noqa: BLE001 - return None, "" - if not isinstance(bucket, dict): - return None, "" - for owner, blob in bucket.items(): - views = (blob or {}).get("views") if isinstance(blob, dict) else None - if isinstance(views, dict) and isinstance(views.get(view_id), dict): - return views[view_id], str(owner) - return None, "" - - -def _mode_of(view: dict) -> str: - """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" - cfg = (view or {}).get("config") if isinstance(view, dict) else None - disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None - return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" - - -def _may_administer_view(session: Session, table_key: str, view_id: str): - """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. - - ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such - attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived - as a bare plain-text 500 rather than as our JSON envelope. - """ - import core.table_store as table_store - - if not table_key.startswith("ut_"): - # The Odoo/registry grids are read-through mirrors with their own permission wall; a - # published projection of one would be a second, secret-gated door onto tenant data whose - # visibility the module gate is supposed to decide. - raise err(400, "not_a_database", "you can publish views on your own databases only") - # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING - # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows - # do not live in the tenant document at all — they are windowed out of the DuckDB mirror - # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, - # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a - # token minted here would resolve to a page that could never render. - # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely - # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the - # MINT — where a person is standing in front of the answer — is the only place that reads as a - # sentence rather than as an empty page. - import core.user_tables as user_tables - - if user_tables.is_connected(table_key, st=session.runtime): - raise err(400, "connected_source", - "a database that reads through a connected source (Odoo, Meta Ads) cannot be " - "published as a public link: its rows are served from the tenant's mirror by a " - "signed-in request, and a public link has no session to serve them with") - view, owner = _find_view(session.runtime, table_key, view_id) - if not isinstance(view, dict): - raise err(404, "no_such_view", "that view no longer exists") - if not (owner == session.uname - or table_store._may_administer(view, session.uname, session.admin)): - raise err(403, "not_yours", "only this view's creator or an admin can publish it") - mode = _mode_of(view) - if mode not in PUBLISHABLE_MODES: - raise err(400, "not_an_interface", - "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " - "published as a link") - cfg = view.get("config") or {} - if cfg.get("cohortLock"): - raise err(400, "reader_scoped", - "this view is locked to a cohort, and a cohort's membership is resolved for " - "the person reading it — a public link has no reader, so the page would show " - "no rows at all. Publish a copy without the cohort lock.") - if _needs_a_reader(cfg.get("filters")): - raise err(400, "reader_scoped", - "this view filters on a cohort, a measure rule or a top-N slice, and each of " - "those is resolved for the person reading it — a public link has no reader, so " - "the page would show no rows at all. Publish a copy filtered on columns.") - return view, mode - - -#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not -#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate -#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What -#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than -#: the view has must SAY SO on the wire, with the cause, never just stop. -PUBLIC_ROW_CAP = 5000 - - -def _needs_a_reader(tree) -> bool: - """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? - - ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the - host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an - answer a single ROW cannot compute … absent, the condition matches NOTHING rather than - everything"*. That default is right (it fails closed) and it is unusable here: an anonymous - page whose every row was silently filtered out is indistinguishable from a broken link, and - the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this - repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` - view would serve zero rows on the server while showing twenty in the browser. - - ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. - That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its - cause, never silently enforced. - - ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would - be a second evaluator that agrees today and drifts the day the leaf shape changes — and it - would drift SILENTLY, because the two answers only differ on views nobody has published yet. - [[one-evaluator-per-question]] - """ - from harness import filter_eval - - def walk(node) -> bool: - if isinstance(node, list): - return any(walk(n) for n in node) - if not isinstance(node, dict): - return False - if node.get("kind") == "group" or isinstance(node.get("children"), list): - return any(walk(n) for n in (node.get("children") or [])) - if filter_eval._is_cohort(node) or filter_eval._is_measure(node): - return True - return node.get("op") in filter_eval.RANK_OPS - - return walk(tree) - - -def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): - """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. - - ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a - link dropped the token and left `config.display.published: true` on the view, so the grid's - own UI would go on saying "published" about a link that no longer resolves. Two records of one - fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with - the writer present and the OTHER half forgotten. - - ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render - without asking; they are a MIRROR, never a source, and nothing in this module reads them back - to decide anything. `read_view_link` deliberately reports both so a drift is visible rather - than assumed away. - - ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly - (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything - that is not the literal `public` stores as `password`) — so a value written here and a value - written by the browser cannot disagree. - """ - def _set(cur): - cur = dict(cur or {}) - for owner, blob in list(cur.items()): - views = (blob or {}).get("views") if isinstance(blob, dict) else None - view = views.get(view_id) if isinstance(views, dict) else None - if not isinstance(view, dict): - continue - cfg = dict(view.get("config") or {}) - disp = dict(cfg.get("display") or {}) - if not disp.get("mode"): - # No display block means no interface view; nothing here should invent one. - continue - if published: - disp["published"] = True - disp["publishAccess"] = "public" if access == "public" else "password" - else: - disp.pop("published", None) - disp.pop("publishAccess", None) - cfg["display"] = disp - cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} - return cur - - try: - rt.update(f"{table_key}_table_workspace", _set, flush="sync") - except Exception: # noqa: BLE001 - # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been - # written. A mirror that failed to update leaves the UI one refresh out of date, which is - # strictly better than a 500 on a publish that actually succeeded. - pass - - -def _entry_for(rt, table_key: str, view_id: str): - """`(token, entry)` for a view's existing link, or `(None, None)`.""" - for token, where in _tokens(rt).items(): - if (isinstance(where, dict) and where.get("table") == table_key - and where.get("view") == view_id): - return str(token), where - return None, None - - -def _state_of(token, entry) -> dict: - """The link's state as the SHARER may see it. Built key by key. - - ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire - to the browser, which is the whole failure this file's bucket exists to avoid, and it would do - it silently the first time somebody added a field. - """ - if not token or not isinstance(entry, dict): - return {"published": False, "access": "password", "token": "", "url": ""} - return { - "published": True, - "access": "public" if entry.get("access") == "public" else "password", - "token": str(token), - "url": _link_of(str(token)), - # A boolean, never the digest and never the salt. - "hasPassphrase": bool(entry.get("pw")), - "createdBy": str(entry.get("createdBy") or ""), - } - - -def _resolve(token: str): - """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. - - ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for - THIS view; it does not mean the holder may read anything else, and nothing downstream of here - may widen the subject beyond the `(table, view)` pair the entry names. - - ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the - form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's - string against every stored token in the deployment. - """ - from harness import runtime as _rt - - if not token or len(token) < 16: - return None - for slug in _rt.known_tenants(): - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - for stored, where in _tokens(rt).items(): - if not _same(str(stored), token) or not isinstance(where, dict): - continue - table_key = str(where.get("table") or "") - view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) - # A view deleted, or re-saved as a grid, since the link was minted. Both answer the - # ONE refusal — "that link is not valid" — rather than explaining which. - if not table_key or not isinstance(view, dict): - return None - if _mode_of(view) not in PUBLISHABLE_MODES: - return None - return rt, slug, table_key, view, where - return None - - -def _coord(value, limit: float): - """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. - - ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put - the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of - this code. Anything that is not a finite number inside the earth's range is not a location and - does not travel. - ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` - is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of - Ghana. - """ - if value is None or isinstance(value, bool): - return None - try: - n = float(value) - except (TypeError, ValueError): - return None - return n if n == n and abs(n) <= limit else None - - -def _visible_keys(view: dict, fields: list) -> list: - """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. - - ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` - builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining - field to it — so `order` is every column the table has, hidden ones included. `visible` is the - only allowlist that exists; there is no stored "hidden" key to subtract. - - ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default - set when a saved view carries none, so a public page that read `[]` as an empty allowlist would - render blank — and one that read it as "all fields" would LEAK. The fallback is the same - predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather - than restated here. - """ - import aios_grid - - by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} - stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] - keep = [k for k in stored if k in by_key] - if keep: - return keep - # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. - # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were - # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — - # and the fallback below then WIDENS the public payload to whatever the table declares by - # default. The publisher chose five columns; the anonymous reader gets the table's idea of - # sensible. That is a widening on the one unauthenticated door in the product. - # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` - # at all (a legacy publish, a view saved before the key existed) has no intent to honour and - # the default IS the right answer for it — that is what the fallback was written for. A view - # that stored five keys and has none left DID state an intent, and every column it named is - # gone: the honest answer is no columns, which renders as an empty published view rather than - # somebody else's data. - if stored: - return [] - try: - default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] - except Exception: # noqa: BLE001 - default_visible = [] - fallback = [str(k) for k in default_visible if str(k) in by_key] - return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] - - -def _public_view(rt, table_key: str, view: dict) -> dict: - """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. - - ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, - `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries - every column the table has. Serialising a row and deleting the fields we do not want inverts - the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an - allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the - shipped precedent and says the same thing about itself. - """ - import core.user_tables as user_tables - from harness import filter_eval - - defn = user_tables.get(table_key, st=rt) or {} - fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] - by_key = {str(f.get("key")): f for f in fields} - keys = _visible_keys(view, fields) - cfg = (view or {}).get("config") or {} - - # The row pool: the table's own rows, narrowed to the fields it declares, exactly as - # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has - # no rows here and is refused at the MINT, so this branch is the only one that can be reached. - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - r = {k: v for k, v in (row or {}).items() if k in by_key} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - - # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already - # refused anything this context could not answer, so an empty result here means the filter - # genuinely matches nothing — not that we failed to resolve it. - ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) - keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, - member_pids=cfg.get("memberPids"))) - chosen = [r for r in rows_src if r.get("pid") in keep] - - # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT - # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads - # the way it does; the first fix I wrote was wrong and the second report proved it. - # - # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string - # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` - # projection and the bypass keyed on the FIELD NAME rather than on the value being a - # coordinate. So a column called `lat` could carry anything — a note, an address — and publish - # it. That is the real defect. - # - # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw - # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column - # list — so gating on `visible` meant an ordinary map published a page with no map, under two - # messages that contradicted each other ("no rows carry a location" vs "this view hides its - # location columns"). - # - # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is - # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a - # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a - # VALUE test, so no naming convention can smuggle anything past it. - # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot - # what the server would not send, and the server must not send what the client would discard. - # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here - # because they sit on opposite sides of a trust boundary, where the server's copy is the wall - # and the client's is display hygiene. - mode = _mode_of(view) - plotted = 0 - if mode == "map": - for r in chosen: - if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: - plotted += 1 - - limits = [] - if mode == "map" and chosen and not plotted: - # R6's second sentence. A map with nothing on it must say WHY — and this says the true - # why, which is about the DATA, because visibility is no longer part of the answer. - limits.append({ - "subject": "map", "effect": "not_plotted", - "detail": f"none of these {len(chosen)} rows carry a usable location", - "recommendation": "add `lat` and `lon` values to the records, then reload this link", - }) - if len(chosen) > PUBLIC_ROW_CAP: - # R6's second sentence. A short page that does not say it is short is the silent - # truncation the rule is actually about. - limits.append({ - "subject": "rows", "effect": "windowed", - "detail": f"this view has {len(chosen)} rows and a published page serves the first " - f"{PUBLIC_ROW_CAP}", - "recommendation": "narrow the view's filters, or share it with named people instead " - "of publishing a link", - }) - chosen = chosen[:PUBLIC_ROW_CAP] - - # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours - # by a column the view HIDES would otherwise put that column's value on every public row — - # the projection leak, arriving through the renderer rather than through the column list. The - # fail-closed choice is to drop the ref and render the map without colour; a published page - # that is slightly plainer beats one that ships a hidden column. - disp_in = (cfg.get("display") or {}) - display = {"mode": mode} - for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): - if disp_in.get(ref) in keys: - display[ref] = disp_in[ref] - - return { - "title": str((view or {}).get("name") or "")[:200], - "mode": display["mode"], - "display": display, - "columns": [{"key": k, - "label": str(by_key[k].get("label") or k), - "type": str(by_key[k].get("type") or "text"), - **({"options": [str(o) for o in by_key[k]["options"]][:200]} - if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") - else {})} - for k in keys], - # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; - # it is a row NUMBER within this table and names nothing outside it. - # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second - # half was missing, which was a LEAK. Found by a verifier that drove this route with a - # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out - # on the wire. - # - # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never - # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds - # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives - # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a - # view can hide them like any other. Hiding them therefore has to work here, because on a - # published page **the projection is the only wall there is** (`ut_*` databases have no - # hidden-field closure behind it, `routes_shares.py`'s docstring). - # - # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it - # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. - # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not - # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the - # names while both values were `None` [[gate-answers-the-wrong-question]]. - # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row - # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, - # and only when they PARSE as coordinates — see the block above for why that is the test. - "rows": [{"pid": r.get("pid"), - **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} - if mode == "map" - and _coord(r.get("lat"), 90) is not None - and _coord(r.get("lon"), 180) is not None else {}), - **{k: r.get(k) for k in keys}} for r in chosen], - "total": len(chosen), - **({"limits": limits} if limits else {}), - } - - -async def _bounded_body(request: Request) -> dict: - """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. - - FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a - declared model is not a bound at all; `content-length` is caller-supplied, so checking it is - not one either. Streaming with a running total is the actual bound. - """ - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > MAX_BODY_BYTES: - raise err(413, "body_too_large", "that request is too large") - chunks.append(chunk) - import json - try: - parsed = json.loads(b"".join(chunks) or b"{}") - except ValueError: - raise err(400, "bad_request", "that request could not be read") - return parsed if isinstance(parsed, dict) else {} - - -# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── -# -# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form -# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` -# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no -# order to get wrong. - - -@router.get("/view-link") -def read_view_link(topic: str = "", view: str = "", - session: Session = Depends(require_session)): - """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" - v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) - token, entry = _entry_for(session.runtime, str(topic), str(view)) - out = _state_of(token, entry) - # The presentational flags contract C1 put on the view spec, echoed back so the client can - # tell whether the two agree. They are a MIRROR of this bucket, never its source. - disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} - out["displayPublished"] = disp.get("published") is True - return out - - -@router.post("/view-link") -async def mint_view_link(request: Request, session: Session = Depends(require_session)): - """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. - - IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not - invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous - one dies in the SAME write, so there is never a window where both open the view. - """ - body = await _bounded_body(request) - topic, view = str(body.get("topic") or ""), str(body.get("view") or "") - _v, _mode = _may_administer_view(session, topic, view) - - access = "public" if body.get("access") == "public" else "password" - raw_pw = body.get("passphrase") - passphrase = "" if raw_pw is None else str(raw_pw) - if len(passphrase) > MAX_PASSPHRASE: - raise err(400, "passphrase_too_long", - f"a passphrase can be at most {MAX_PASSPHRASE} characters") - - existing_token, existing = _entry_for(session.runtime, topic, view) - rotate = bool(body.get("rotate")) - fresh = secrets.token_urlsafe(TOKEN_BYTES) - - # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: - # the caller supplied a passphrase now, or one was already stored and is being kept. Anything - # else is refused HERE rather than stored and refused later — a link that cannot be opened by - # anybody is not a safe default, it is a broken feature that reads as a permission bug. - if access == "password": - if passphrase and len(passphrase) < MIN_PASSPHRASE: - raise err(400, "passphrase_too_short", - f"a passphrase needs at least {MIN_PASSPHRASE} characters") - if not passphrase and not (existing or {}).get("pw"): - raise err(400, "passphrase_required", - "a password-protected link needs a passphrase") - - minted = {"token": existing_token or fresh} - - def _set(cur): - cur = dict(cur or {}) - prev = None - for stored, where in list(cur.items()): - if (isinstance(where, dict) and where.get("table") == topic - and where.get("view") == view): - prev = dict(where) - cur.pop(stored, None) - if not rotate: - minted["token"] = str(stored) - if rotate or prev is None: - minted["token"] = fresh - entry = {"table": topic, "view": view, "access": access, - "createdBy": str((prev or {}).get("createdBy") or session.uname), - "createdAt": float((prev or {}).get("createdAt") or time.time())} - # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED - # it — found by a verifier that traced the toggle rather than the happy path. Publishing - # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was - # kept; switching back to `password` then demanded a new passphrase, silently, while the - # panel's own sentence promised the opposite ("leave blank to keep the current one"). - # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` - # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, - # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI - # makes and the store breaks is a defect either way, and the honest fix is to keep the - # promise rather than to reword it. - if passphrase: - salt = secrets.token_bytes(PW_SALT_BYTES) - entry["salt"] = salt.hex() - entry["iter"] = PW_ITERATIONS - entry["pw"] = _hash_pw(passphrase, salt) - elif (prev or {}).get("pw"): - # Carried key by key, so a future field on the entry is not silently inherited. - entry["salt"] = str((prev or {}).get("salt") or "") - entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) - entry["pw"] = str((prev or {}).get("pw") or "") - cur[minted["token"]] = entry - return cur - - session.runtime.update(TOKENS_KEY, _set, flush="sync") - _mirror_display(session.runtime, topic, view, True, access) - token, entry = _entry_for(session.runtime, topic, view) - # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered - # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that - # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back - # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. - # If the write cannot be read back, say so; do not describe a state nobody verified. - if not entry: - raise err(503, "not_saved", - "the link was minted but could not be read back — reload and try again") - return _state_of(token, entry) - - -@router.delete("/view-link") -async def revoke_view_link(request: Request, session: Session = Depends(require_session)): - """Unpublish. `{topic, view}`. - - ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the - index in this write, so the old link stops resolving immediately; and because a later publish - mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An - implementation that kept the token and flipped an `enabled` flag would leave the secret live - in the store, one bug away from working again, and would make "revoked" a property somebody - could forget to check on a code path added later. - """ - body = await _bounded_body(request) - topic, view = str(body.get("topic") or ""), str(body.get("view") or "") - _may_administer_view(session, topic, view) - - def _drop(cur): - cur = dict(cur or {}) - for stored, where in list(cur.items()): - if (isinstance(where, dict) and where.get("table") == topic - and where.get("view") == view): - cur.pop(stored, None) - return cur - - session.runtime.update(TOKENS_KEY, _drop, flush="sync") - # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing - # "published" about a link that no longer resolves — the half a reviewer caught. - _mirror_display(session.runtime, topic, view, False) - return {"published": False, "access": "password", "token": "", "url": ""} - - -# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── -# -# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not -# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a -# route is public exactly by omitting the dependency. Which is why the two are kept together, -# under one banner, rather than filed beside the sharer's routes they resemble. - - -@router.get("/published/{token}") -def get_published(token: str, request: Request): - """The published view. Read-only, unauthenticated, projected to the view's own columns.""" - if not _rate_ok(_client_ip(request), time.time()): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - found = _resolve(token) - if not found: - # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. - # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, - # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own - # docstring says the difference between "no such link" and "that link was revoked" must - # never be observable — and the route beside it published that difference in its status - # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the - # passphrase is all that is left and it can be attacked offline-cheap. - # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no - # title, no columns, no count, the same bytes a real password link returns before anyone - # has tried to open it. The guess then costs a POST with a passphrase, which is rate - # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public - # link is for; what stops being visible is which PASSWORD tokens exist. - _note_failure(_client_ip(request), time.time()) - _equalise_pw_cost(None) - return {"locked": True} - rt, _slug, table_key, view, entry = found - if entry.get("access") == "password": - # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render - # a passphrase prompt, so this 200 says "there is something here and it is locked" and - # NOTHING else — no title, no column names, no row count. A 403 here would be - # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for - # a link the holder has not tried to open yet. - return {"locked": True} - return _public_view(rt, table_key, view) - - -@router.post("/published/{token}") -async def open_published(token: str, request: Request): - """Open a password-protected link. `{passphrase}`. - - ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they - differed, the route would confirm which tokens are real to anybody willing to send one guess — - and a 24-byte token's entire protection is that it cannot be found by guessing. - ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never - stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and - anyone may claim an identity; this is AUTHENTICATION and is treated as one. - """ - now = time.time() - if not _rate_ok(_client_ip(request), now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - body = await _bounded_body(request) - found = _resolve(token) - if not found: - # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and - # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 - # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. - # Both arms now cost the same, so the identical 403 above is finally identical in practice - # rather than only in wording. - _note_failure(_client_ip(request), now) - _equalise_pw_cost(None) - raise _refuse() - rt, _slug, table_key, view, entry = found - if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): - _note_failure(_client_ip(request), now) - raise _refuse() - # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the - # clock on a route whose whole job is to be uninformative. - if entry.get("access") != "password": - _equalise_pw_cost(entry) - return _public_view(rt, table_key, view) +"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). + +The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface +(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed +publicly or with a password that the user that share it can toggle."* + +R5, in five clauses, and every one of them is load-bearing: + * publish = a per-view secret token in a **server-only bucket**; + * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; + * an **unauthenticated** read-only route plus a `#/v/` client route; + * the publish surface **projects only the columns the view shows** — never the whole table; + * **creator-or-admin** may publish, and **revoke ROTATES** the token. + +⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door +has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal +for every resolution failure (so the route is not an oracle for which tokens are real), a +constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the +whole body before the handler's first line, and `content-length` is caller-supplied), a sliding +rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. + +⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE +door into a table; a publish token buys a READ projection of one view. Folding them into one index +would make a single leaked string ambiguous about which of the two it opens, and would make +`_resolve` return an object whose capability depends on a field rather than on which door was +knocked. Two buckets, two resolvers, one shape. + +⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and +they are not the same question (audit S-4): the grant registry drives *Shared with me*, +`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing +— an unauthenticated read of a projection, bound to a secret rather than to an account. A published +link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot +be revoked by removing a person, because there is no person. + + python verify_forms.py # this file's gate; section 6 onward +""" +import hashlib +import hmac +import os +import secrets +import time + +from fastapi import APIRouter, Depends, Request + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. +#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec +#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser +#: writes that spec on every autosave — so anything secret living there would be echoed back to the +#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. +TOKENS_KEY = "publish_tokens" +TOKEN_BYTES = 24 + +#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. +#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says +#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is +#: not a stronger string, it is that the secret is never stored, never logged and never echoed. +PW_ITERATIONS = 240_000 +PW_SALT_BYTES = 16 +MIN_PASSPHRASE = 6 +MAX_PASSPHRASE = 128 + +#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so +#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. +RATE_WINDOW_S = 60 +RATE_PER_WINDOW = 30 +MAX_BODY_BYTES = 16 * 1024 + +_HITS: dict = {} + +#: The view kinds that may be published, i.e. R5's "interface". +#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 +#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two +#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a +#: server list that silently drifts from the picker means a mode a user can create and cannot +#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row +#: set; publishing one would be publishing the table, which is exactly what R5's projection clause +#: exists to prevent. +#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. +#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a +#: row set the publisher already curated; a form's row set is other people's answers, gathered under +#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into +#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the +#: product, with no wall between the link and the data. +#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which +#: serves the BLANK form for submitting and never the stored rows. So this is not a capability +#: removed, it is a second door onto the same object that should never have existed beside the +#: first. Publishing a form to be filled in still works, at the URL that was always for it. +#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not +#: offer Publish on a form view, or the picker promises what this refuses. +PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") + + +def _refuse(): + """THE ONE REFUSAL, for every way a published link can fail to resolve. + + Wrong token, revoked link, deleted view, deleted table, a view that stopped being an + interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: + the difference between "no such link" and "that link was revoked" tells an enumerator which + tokens are real, and the difference between "no such link" and "wrong passphrase" tells them + which links are worth guessing at. + """ + return err(403, "bad_publish_token", "that link is not valid") + + +def _same(a: str, b: str) -> bool: + """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" + return hmac.compare_digest(str(a or ""), str(b or "")) + + +def _client_ip(request: Request) -> str: + """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. + + The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new + value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted + through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its + old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the + use it could not support. It is safe as a LOG field and as nothing else. + + The socket peer is the only thing here the caller cannot choose, so it is the key. + ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES + ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm + that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer + is that we cannot separate callers, not that we should throttle them together. + """ + return request.client.host if request.client else "?" + + +def _rate_ok(key: str, now: float) -> bool: + """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. + + A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the + whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire + protection is that guessing a 24-byte token is slow. + + ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer + survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at + all; an enumerator produces nothing but misses. Counting every request under one shared peer + would have denied service to everybody the moment one guesser showed up — trading a token + oracle for an outage is not a fix. + """ + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + _HITS[key] = seen + return len(seen) < RATE_PER_WINDOW + + +def _note_failure(key: str, now: float) -> None: + """Record one failed resolution against this peer, and keep the table bounded.""" + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + + +#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash +#: to check against. Its value is irrelevant; its COST is the point. +_DUMMY_SALT = b"\x00" * 16 + + +def _equalise_pw_cost(entry) -> None: + """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. + + MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in + ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording + ("that link is not valid", identical for both) was undone by the clock: anyone could sort real + tokens from fake ones by timing alone, then spend their guesses only on the real ones. The + docstring on `_refuse` describes exactly the leak the code then handed over for free. + + Called on every path that fails BEFORE a passphrase check would have happened, so the cheap + branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so + a link minted under a different iteration count stays indistinguishable too. + """ + iterations = PW_ITERATIONS + if isinstance(entry, dict): + try: + iterations = int(entry.get("iter") or PW_ITERATIONS) + except (TypeError, ValueError): + iterations = PW_ITERATIONS + _hash_pw("", _DUMMY_SALT, iterations) + + +def _public_base() -> str: + return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") + + +def _link_of(token: str) -> str: + return f"{_public_base()}/#/v/{token}" + + +def _tokens(rt) -> dict: + """This tenant's publish index, always a dict.""" + try: + found = rt.get(TOKENS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: + return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), + salt, int(iterations)).hex() + + +def _pw_ok(entry: dict, passphrase: str) -> bool: + """Constant-time verify against the stored digest. + + ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" + from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not + fall open. [[default-must-pass-its-own-guard]] + """ + stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") + if not stored or not salt_hex: + return False + try: + salt = bytes.fromhex(salt_hex) + except ValueError: + return False + got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) + return _same(got, stored) + + +def _find_view(rt, table_key: str, view_id: str): + """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. + + The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` + does, and the reason a view is addressable by id ALONE here: the id is unique within the table, + while the owner is what we are trying to discover. + """ + if not table_key or not view_id: + return None, "" + try: + bucket = rt.get(f"{table_key}_table_workspace") or {} + except Exception: # noqa: BLE001 + return None, "" + if not isinstance(bucket, dict): + return None, "" + for owner, blob in bucket.items(): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + if isinstance(views, dict) and isinstance(views.get(view_id), dict): + return views[view_id], str(owner) + return None, "" + + +def _mode_of(view: dict) -> str: + """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" + cfg = (view or {}).get("config") if isinstance(view, dict) else None + disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None + return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" + + +def _may_administer_view(session: Session, table_key: str, view_id: str): + """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. + + ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such + attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived + as a bare plain-text 500 rather than as our JSON envelope. + """ + import core.table_store as table_store + + if not table_key.startswith("ut_"): + # The Odoo/registry grids are read-through mirrors with their own permission wall; a + # published projection of one would be a second, secret-gated door onto tenant data whose + # visibility the module gate is supposed to decide. + raise err(400, "not_a_database", "you can publish views on your own databases only") + # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING + # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows + # do not live in the tenant document at all — they are windowed out of the DuckDB mirror + # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, + # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a + # token minted here would resolve to a page that could never render. + # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely + # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the + # MINT — where a person is standing in front of the answer — is the only place that reads as a + # sentence rather than as an empty page. + import core.user_tables as user_tables + + if user_tables.is_connected(table_key, st=session.runtime): + raise err(400, "connected_source", + "a database that reads through a connected source (Odoo, Meta Ads) cannot be " + "published as a public link: its rows are served from the tenant's mirror by a " + "signed-in request, and a public link has no session to serve them with") + view, owner = _find_view(session.runtime, table_key, view_id) + if not isinstance(view, dict): + raise err(404, "no_such_view", "that view no longer exists") + if not (owner == session.uname + or table_store._may_administer(view, session.uname, session.admin)): + raise err(403, "not_yours", "only this view's creator or an admin can publish it") + mode = _mode_of(view) + if mode not in PUBLISHABLE_MODES: + raise err(400, "not_an_interface", + "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " + "published as a link") + cfg = view.get("config") or {} + if cfg.get("cohortLock"): + raise err(400, "reader_scoped", + "this view is locked to a cohort, and a cohort's membership is resolved for " + "the person reading it — a public link has no reader, so the page would show " + "no rows at all. Publish a copy without the cohort lock.") + if _needs_a_reader(cfg.get("filters")): + raise err(400, "reader_scoped", + "this view filters on a cohort, a measure rule or a top-N slice, and each of " + "those is resolved for the person reading it — a public link has no reader, so " + "the page would show no rows at all. Publish a copy filtered on columns.") + return view, mode + + +#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not +#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate +#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What +#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than +#: the view has must SAY SO on the wire, with the cause, never just stop. +PUBLIC_ROW_CAP = 5000 + + +def _needs_a_reader(tree) -> bool: + """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? + + ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the + host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an + answer a single ROW cannot compute … absent, the condition matches NOTHING rather than + everything"*. That default is right (it fails closed) and it is unusable here: an anonymous + page whose every row was silently filtered out is indistinguishable from a broken link, and + the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this + repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` + view would serve zero rows on the server while showing twenty in the browser. + + ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. + That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its + cause, never silently enforced. + + ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would + be a second evaluator that agrees today and drifts the day the leaf shape changes — and it + would drift SILENTLY, because the two answers only differ on views nobody has published yet. + [[one-evaluator-per-question]] + """ + from harness import filter_eval + + def walk(node) -> bool: + if isinstance(node, list): + return any(walk(n) for n in node) + if not isinstance(node, dict): + return False + if node.get("kind") == "group" or isinstance(node.get("children"), list): + return any(walk(n) for n in (node.get("children") or [])) + if filter_eval._is_cohort(node) or filter_eval._is_measure(node): + return True + return node.get("op") in filter_eval.RANK_OPS + + return walk(tree) + + +def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): + """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. + + ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a + link dropped the token and left `config.display.published: true` on the view, so the grid's + own UI would go on saying "published" about a link that no longer resolves. Two records of one + fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with + the writer present and the OTHER half forgotten. + + ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render + without asking; they are a MIRROR, never a source, and nothing in this module reads them back + to decide anything. `read_view_link` deliberately reports both so a drift is visible rather + than assumed away. + + ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly + (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything + that is not the literal `public` stores as `password`) — so a value written here and a value + written by the browser cannot disagree. + """ + def _set(cur): + cur = dict(cur or {}) + for owner, blob in list(cur.items()): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + view = views.get(view_id) if isinstance(views, dict) else None + if not isinstance(view, dict): + continue + cfg = dict(view.get("config") or {}) + disp = dict(cfg.get("display") or {}) + if not disp.get("mode"): + # No display block means no interface view; nothing here should invent one. + continue + if published: + disp["published"] = True + disp["publishAccess"] = "public" if access == "public" else "password" + else: + disp.pop("published", None) + disp.pop("publishAccess", None) + cfg["display"] = disp + cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} + return cur + + try: + rt.update(f"{table_key}_table_workspace", _set, flush="sync") + except Exception: # noqa: BLE001 + # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been + # written. A mirror that failed to update leaves the UI one refresh out of date, which is + # strictly better than a 500 on a publish that actually succeeded. + pass + + +def _entry_for(rt, table_key: str, view_id: str): + """`(token, entry)` for a view's existing link, or `(None, None)`.""" + for token, where in _tokens(rt).items(): + if (isinstance(where, dict) and where.get("table") == table_key + and where.get("view") == view_id): + return str(token), where + return None, None + + +def _state_of(token, entry) -> dict: + """The link's state as the SHARER may see it. Built key by key. + + ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire + to the browser, which is the whole failure this file's bucket exists to avoid, and it would do + it silently the first time somebody added a field. + """ + if not token or not isinstance(entry, dict): + return {"published": False, "access": "password", "token": "", "url": ""} + return { + "published": True, + "access": "public" if entry.get("access") == "public" else "password", + "token": str(token), + "url": _link_of(str(token)), + # A boolean, never the digest and never the salt. + "hasPassphrase": bool(entry.get("pw")), + "createdBy": str(entry.get("createdBy") or ""), + } + + +def _resolve(token: str): + """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. + + ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for + THIS view; it does not mean the holder may read anything else, and nothing downstream of here + may widen the subject beyond the `(table, view)` pair the entry names. + + ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the + form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's + string against every stored token in the deployment. + """ + from harness import runtime as _rt + + if not token or len(token) < 16: + return None + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + for stored, where in _tokens(rt).items(): + if not _same(str(stored), token) or not isinstance(where, dict): + continue + table_key = str(where.get("table") or "") + view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) + # A view deleted, or re-saved as a grid, since the link was minted. Both answer the + # ONE refusal — "that link is not valid" — rather than explaining which. + if not table_key or not isinstance(view, dict): + return None + if _mode_of(view) not in PUBLISHABLE_MODES: + return None + return rt, slug, table_key, view, where + return None + + +def _coord(value, limit: float): + """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. + + ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put + the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of + this code. Anything that is not a finite number inside the earth's range is not a location and + does not travel. + ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` + is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of + Ghana. + """ + if value is None or isinstance(value, bool): + return None + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if n == n and abs(n) <= limit else None + + +def _visible_keys(view: dict, fields: list) -> list: + """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. + + ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` + builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining + field to it — so `order` is every column the table has, hidden ones included. `visible` is the + only allowlist that exists; there is no stored "hidden" key to subtract. + + ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default + set when a saved view carries none, so a public page that read `[]` as an empty allowlist would + render blank — and one that read it as "all fields" would LEAK. The fallback is the same + predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather + than restated here. + """ + import aios_grid + + by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} + stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] + keep = [k for k in stored if k in by_key] + if keep: + return keep + # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. + # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were + # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — + # and the fallback below then WIDENS the public payload to whatever the table declares by + # default. The publisher chose five columns; the anonymous reader gets the table's idea of + # sensible. That is a widening on the one unauthenticated door in the product. + # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` + # at all (a legacy publish, a view saved before the key existed) has no intent to honour and + # the default IS the right answer for it — that is what the fallback was written for. A view + # that stored five keys and has none left DID state an intent, and every column it named is + # gone: the honest answer is no columns, which renders as an empty published view rather than + # somebody else's data. + if stored: + return [] + try: + default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] + except Exception: # noqa: BLE001 + default_visible = [] + fallback = [str(k) for k in default_visible if str(k) in by_key] + return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] + + +def _public_view(rt, table_key: str, view: dict) -> dict: + """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. + + ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, + `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries + every column the table has. Serialising a row and deleting the fields we do not want inverts + the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an + allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the + shipped precedent and says the same thing about itself. + """ + import core.user_tables as user_tables + from harness import filter_eval + + defn = user_tables.get(table_key, st=rt) or {} + fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] + by_key = {str(f.get("key")): f for f in fields} + keys = _visible_keys(view, fields) + cfg = (view or {}).get("config") or {} + + # The row pool: the table's own rows, narrowed to the fields it declares, exactly as + # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has + # no rows here and is refused at the MINT, so this branch is the only one that can be reached. + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in by_key} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + + # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already + # refused anything this context could not answer, so an empty result here means the filter + # genuinely matches nothing — not that we failed to resolve it. + ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) + keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, + member_pids=cfg.get("memberPids"))) + chosen = [r for r in rows_src if r.get("pid") in keep] + + # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT + # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads + # the way it does; the first fix I wrote was wrong and the second report proved it. + # + # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string + # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` + # projection and the bypass keyed on the FIELD NAME rather than on the value being a + # coordinate. So a column called `lat` could carry anything — a note, an address — and publish + # it. That is the real defect. + # + # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw + # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column + # list — so gating on `visible` meant an ordinary map published a page with no map, under two + # messages that contradicted each other ("no rows carry a location" vs "this view hides its + # location columns"). + # + # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is + # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a + # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a + # VALUE test, so no naming convention can smuggle anything past it. + # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot + # what the server would not send, and the server must not send what the client would discard. + # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here + # because they sit on opposite sides of a trust boundary, where the server's copy is the wall + # and the client's is display hygiene. + mode = _mode_of(view) + plotted = 0 + if mode == "map": + for r in chosen: + if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: + plotted += 1 + + limits = [] + if mode == "map" and chosen and not plotted: + # R6's second sentence. A map with nothing on it must say WHY — and this says the true + # why, which is about the DATA, because visibility is no longer part of the answer. + limits.append({ + "subject": "map", "effect": "not_plotted", + "detail": f"none of these {len(chosen)} rows carry a usable location", + "recommendation": "add `lat` and `lon` values to the records, then reload this link", + }) + if len(chosen) > PUBLIC_ROW_CAP: + # R6's second sentence. A short page that does not say it is short is the silent + # truncation the rule is actually about. + limits.append({ + "subject": "rows", "effect": "windowed", + "detail": f"this view has {len(chosen)} rows and a published page serves the first " + f"{PUBLIC_ROW_CAP}", + "recommendation": "narrow the view's filters, or share it with named people instead " + "of publishing a link", + }) + chosen = chosen[:PUBLIC_ROW_CAP] + + # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours + # by a column the view HIDES would otherwise put that column's value on every public row — + # the projection leak, arriving through the renderer rather than through the column list. The + # fail-closed choice is to drop the ref and render the map without colour; a published page + # that is slightly plainer beats one that ships a hidden column. + disp_in = (cfg.get("display") or {}) + display = {"mode": mode} + for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): + if disp_in.get(ref) in keys: + display[ref] = disp_in[ref] + + return { + "title": str((view or {}).get("name") or "")[:200], + "mode": display["mode"], + "display": display, + "columns": [{"key": k, + "label": str(by_key[k].get("label") or k), + "type": str(by_key[k].get("type") or "text"), + **({"options": [str(o) for o in by_key[k]["options"]][:200]} + if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") + else {})} + for k in keys], + # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; + # it is a row NUMBER within this table and names nothing outside it. + # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second + # half was missing, which was a LEAK. Found by a verifier that drove this route with a + # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out + # on the wire. + # + # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never + # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds + # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives + # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a + # view can hide them like any other. Hiding them therefore has to work here, because on a + # published page **the projection is the only wall there is** (`ut_*` databases have no + # hidden-field closure behind it, `routes_shares.py`'s docstring). + # + # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it + # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. + # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not + # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the + # names while both values were `None` [[gate-answers-the-wrong-question]]. + # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row + # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, + # and only when they PARSE as coordinates — see the block above for why that is the test. + "rows": [{"pid": r.get("pid"), + **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} + if mode == "map" + and _coord(r.get("lat"), 90) is not None + and _coord(r.get("lon"), 180) is not None else {}), + **{k: r.get(k) for k in keys}} for r in chosen], + "total": len(chosen), + **({"limits": limits} if limits else {}), + } + + +async def _bounded_body(request: Request) -> dict: + """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. + + FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a + declared model is not a bound at all; `content-length` is caller-supplied, so checking it is + not one either. Streaming with a running total is the actual bound. + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + import json + try: + parsed = json.loads(b"".join(chunks) or b"{}") + except ValueError: + raise err(400, "bad_request", "that request could not be read") + return parsed if isinstance(parsed, dict) else {} + + +# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── +# +# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form +# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` +# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no +# order to get wrong. + + +@router.get("/view-link") +def read_view_link(topic: str = "", view: str = "", + session: Session = Depends(require_session)): + """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" + v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) + token, entry = _entry_for(session.runtime, str(topic), str(view)) + out = _state_of(token, entry) + # The presentational flags contract C1 put on the view spec, echoed back so the client can + # tell whether the two agree. They are a MIRROR of this bucket, never its source. + disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} + out["displayPublished"] = disp.get("published") is True + return out + + +@router.post("/view-link") +async def mint_view_link(request: Request, session: Session = Depends(require_session)): + """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. + + IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not + invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous + one dies in the SAME write, so there is never a window where both open the view. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _v, _mode = _may_administer_view(session, topic, view) + + access = "public" if body.get("access") == "public" else "password" + raw_pw = body.get("passphrase") + passphrase = "" if raw_pw is None else str(raw_pw) + if len(passphrase) > MAX_PASSPHRASE: + raise err(400, "passphrase_too_long", + f"a passphrase can be at most {MAX_PASSPHRASE} characters") + + existing_token, existing = _entry_for(session.runtime, topic, view) + rotate = bool(body.get("rotate")) + fresh = secrets.token_urlsafe(TOKEN_BYTES) + + # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: + # the caller supplied a passphrase now, or one was already stored and is being kept. Anything + # else is refused HERE rather than stored and refused later — a link that cannot be opened by + # anybody is not a safe default, it is a broken feature that reads as a permission bug. + if access == "password": + if passphrase and len(passphrase) < MIN_PASSPHRASE: + raise err(400, "passphrase_too_short", + f"a passphrase needs at least {MIN_PASSPHRASE} characters") + if not passphrase and not (existing or {}).get("pw"): + raise err(400, "passphrase_required", + "a password-protected link needs a passphrase") + + minted = {"token": existing_token or fresh} + + def _set(cur): + cur = dict(cur or {}) + prev = None + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + prev = dict(where) + cur.pop(stored, None) + if not rotate: + minted["token"] = str(stored) + if rotate or prev is None: + minted["token"] = fresh + entry = {"table": topic, "view": view, "access": access, + "createdBy": str((prev or {}).get("createdBy") or session.uname), + "createdAt": float((prev or {}).get("createdAt") or time.time())} + # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED + # it — found by a verifier that traced the toggle rather than the happy path. Publishing + # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was + # kept; switching back to `password` then demanded a new passphrase, silently, while the + # panel's own sentence promised the opposite ("leave blank to keep the current one"). + # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` + # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, + # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI + # makes and the store breaks is a defect either way, and the honest fix is to keep the + # promise rather than to reword it. + if passphrase: + salt = secrets.token_bytes(PW_SALT_BYTES) + entry["salt"] = salt.hex() + entry["iter"] = PW_ITERATIONS + entry["pw"] = _hash_pw(passphrase, salt) + elif (prev or {}).get("pw"): + # Carried key by key, so a future field on the entry is not silently inherited. + entry["salt"] = str((prev or {}).get("salt") or "") + entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) + entry["pw"] = str((prev or {}).get("pw") or "") + cur[minted["token"]] = entry + return cur + + session.runtime.update(TOKENS_KEY, _set, flush="sync") + _mirror_display(session.runtime, topic, view, True, access) + token, entry = _entry_for(session.runtime, topic, view) + # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered + # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that + # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back + # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. + # If the write cannot be read back, say so; do not describe a state nobody verified. + if not entry: + raise err(503, "not_saved", + "the link was minted but could not be read back — reload and try again") + return _state_of(token, entry) + + +@router.delete("/view-link") +async def revoke_view_link(request: Request, session: Session = Depends(require_session)): + """Unpublish. `{topic, view}`. + + ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the + index in this write, so the old link stops resolving immediately; and because a later publish + mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An + implementation that kept the token and flipped an `enabled` flag would leave the secret live + in the store, one bug away from working again, and would make "revoked" a property somebody + could forget to check on a code path added later. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _may_administer_view(session, topic, view) + + def _drop(cur): + cur = dict(cur or {}) + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + cur.pop(stored, None) + return cur + + session.runtime.update(TOKENS_KEY, _drop, flush="sync") + # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing + # "published" about a link that no longer resolves — the half a reviewer caught. + _mirror_display(session.runtime, topic, view, False) + return {"published": False, "access": "password", "token": "", "url": ""} + + +# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── +# +# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not +# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a +# route is public exactly by omitting the dependency. Which is why the two are kept together, +# under one banner, rather than filed beside the sharer's routes they resemble. + + +@router.get("/published/{token}") +def get_published(token: str, request: Request): + """The published view. Read-only, unauthenticated, projected to the view's own columns.""" + if not _rate_ok(_client_ip(request), time.time()): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. + # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, + # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own + # docstring says the difference between "no such link" and "that link was revoked" must + # never be observable — and the route beside it published that difference in its status + # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the + # passphrase is all that is left and it can be attacked offline-cheap. + # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no + # title, no columns, no count, the same bytes a real password link returns before anyone + # has tried to open it. The guess then costs a POST with a passphrase, which is rate + # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public + # link is for; what stops being visible is which PASSWORD tokens exist. + _note_failure(_client_ip(request), time.time()) + _equalise_pw_cost(None) + return {"locked": True} + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password": + # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render + # a passphrase prompt, so this 200 says "there is something here and it is locked" and + # NOTHING else — no title, no column names, no row count. A 403 here would be + # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for + # a link the holder has not tried to open yet. + return {"locked": True} + return _public_view(rt, table_key, view) + + +@router.post("/published/{token}") +async def open_published(token: str, request: Request): + """Open a password-protected link. `{passphrase}`. + + ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they + differed, the route would confirm which tokens are real to anybody willing to send one guess — + and a 24-byte token's entire protection is that it cannot be found by guessing. + ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never + stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and + anyone may claim an identity; this is AUTHENTICATION and is treated as one. + """ + now = time.time() + if not _rate_ok(_client_ip(request), now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + body = await _bounded_body(request) + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and + # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 + # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. + # Both arms now cost the same, so the identical 403 above is finally identical in practice + # rather than only in wording. + _note_failure(_client_ip(request), now) + _equalise_pw_cost(None) + raise _refuse() + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): + _note_failure(_client_ip(request), now) + raise _refuse() + # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the + # clock on a route whose whole job is to be uninformative. + if entry.get("access") != "password": + _equalise_pw_cost(entry) + return _public_view(rt, table_key, view) diff --git a/api/routes_query.py b/api/routes_query.py index cfa0e6b44d6d1b7ddebde412fd4d18e7886b0f8f..5ba78e96bada42a91bd11c06ba1371ca7b483b21 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -1,1462 +1,1706 @@ -"""Assistant chat and Query-owned virtual artefacts (W33-T72 / C8). - -Query reads only the detached, permission-filtered snapshot from -``deps.assistant_read_scope``. It owns a separate per-user namespace for threads, messages, -citations and virtual views; it never opens or mutates a source workspace. -""" -import copy -import csv -import datetime as _dt -import hashlib -import io -import json -import os -import re -import uuid - -from fastapi import APIRouter, Body, Depends, Response - -from deps import (Session, assistant_read_scope, assistant_source_status, err, - require_session) - -router = APIRouter(prefix="/api/v1") - -MAX_QUESTION = 500 -MAX_VISIBLE = 24 -MAX_ARTIFACTS = 200 -# ⭐⭐ R16 — THE THREAD IS REPLAYED, AND THESE TWO NUMBERS ARE THE WHOLE BOUND ON IT. -# The chat was STATELESS: `_call_model` built `[system, user(question)]` and the conversation on -# screen reached the model as nothing at all, so "what about the other one?" could not be answered -# and the assistant could not grill anybody about anything. Replay is bounded twice because the -# two limits fail differently: a turn COUNT stops a long chat costing more every turn, and a -# CHARACTER budget stops one pasted wall of text from being the whole context. Both count from the -# NEWEST end, because that is what a pronoun refers to. -MAX_HISTORY_TURNS = 12 -MAX_HISTORY_CHARS = 6000 -MAX_RATING_REASON = 200 -QUERY_RATINGS = ("up", "down") -MODEL_AUTO = "auto" -# ⛔⛔ WHY QUERY DECLARES THE ENV NAMES RATHER THAN READING `analyst.PROVIDERS`: MEASURED -# 2026-08-16, `import harness.analyst` costs **5 to 8 SECONDS**, warm, on this box (it pulls -# `harness.tools`, which pulls yaml and the skill recipes). `GET /query` is a LIST path that BOTH -# the Assistant and Query call on mount, so importing the ladder to answer "which models can this -# deployment actually call?" would put eight seconds on the first paint of the AI module. The map -# below is Query's own declaration, and it is deliberately the cheap half. -# ⚠ A SECOND DECLARATION IS A DIVERGENCE WAITING TO HAPPEN, so it is CHECKED rather than trusted: -# `_providers()` already holds the platform ladder in memory and reconciles there, and anything it -# finds is REPORTED into the payload instead of a provider silently never being offered. -# The ORDER is Query's own and is deliberate: cerebras first, because this path needs tool calling -# and cerebras carries this account's tool-capable model. -QUERY_PROVIDER_ENV = { - "cerebras": "CEREBRAS_API_KEY", - "groq": "GROQ_API_KEY", - "openrouter": "OPENROUTER_API_KEY", -} -QUERY_PROVIDER_ORDER = tuple(QUERY_PROVIDER_ENV) -# Filled the first time the platform ladder is loaded; empty means "not yet known", never "clean". -_LADDER_DRIFT = [] -QUERY_KINDS = ("grid", "chart", "calendar", "kanban", "timeseries", "map", "list") -QUERY_WORKSPACE_EVENTS = {"view_create", "view_upsert", "view_delete"} -QUERY_EXCLUDED = { - "form": "Forms collect new data and are not a read-only Query artefact.", - "catalog": "Catalog is a source-native presentation that Query cannot safely mutate.", - "swipe": "Swipe is an interactive source-native presentation, not an Assistant output.", -} -_MODE_REFS = { - "kanban": [("stackField", ("select",), True)], - "calendar": [("dateField", ("date",), True)], - "timeseries": [("dateField", ("date",), True)], - "map": [("colorField", ("select",), False), ("sizeField", ("int", "currency", "pct"), False)], -} - - -def _now_iso(): - return _dt.datetime.now(_dt.timezone.utc).isoformat() - - -def _grid(): - import aios_grid - return aios_grid - - -def _safe(value): - """Detach values before they enter a durable Query object or an API reply.""" - return json.loads(json.dumps(value, default=str)) - - -def _namespace_key(session): - """A tenant runtime has one isolated key for each user's Query-owned objects.""" - principal = f"{session.tenant}:{session.uname}".encode("utf-8") - return "query_user_" + hashlib.sha256(principal).hexdigest()[:24] - - -def _blank_state(): - return {"version": 1, "threads": {}, "messages": {}, "citations": {}, "views": {}} - - -def _state(session): - try: - raw = session.runtime.get(_namespace_key(session)) or {} - except Exception: - raw = {} - if not isinstance(raw, dict): - return _blank_state() - out = _blank_state() - for key in out: - if key == "version": - continue - if isinstance(raw.get(key), dict): - out[key] = copy.deepcopy(raw[key]) - return out - - -def _new_id(prefix): - return f"{prefix}_{uuid.uuid4().hex[:16]}" - - -def model_choices(): - """Choices are stable even when one is not configured, so explicit means explicit.""" - return [MODEL_AUTO, *QUERY_PROVIDER_ORDER] - - -def model_status(): - """Per choice: can this deployment actually CALL it, and if not, why not. - - ⭐⭐ THE SAME DEFECT THE SOURCE CHIPS WERE FIXED FOR IN WAVE 33, ONE CONTROL TO THE LEFT. - `permitted` is not `answerable` for a database; OFFERED is not CALLABLE for a model. The picker - listed every name in the ladder whether or not this deployment holds its key, so choosing one - spent a click and a turn to be told "the selected model is unavailable" by the server, which - knew before the click. The reason travels with the flag, exactly as `sources` does. - - ⚠ It names no environment variable. "This model is not configured on this deployment" is the - cause a tenant user can act on (ask an admin); the variable name is our deployment's business. - """ - live = [name for name in QUERY_PROVIDER_ORDER if os.environ.get(QUERY_PROVIDER_ENV[name])] - rows = [{"model": MODEL_AUTO, "available": bool(live), - "reason": "" if live else "no model is configured on this deployment"}] - for name in QUERY_PROVIDER_ORDER: - rows.append({"model": name, "available": name in live, - "reason": "" if name in live - else "this model is not configured on this deployment"}) - # Anything the platform ladder carries that Query does not offer, once a chat has taught us. - rows.extend({"model": name, "available": False, - "reason": "the platform can reach this model but Query does not offer it yet"} - for name in _LADDER_DRIFT) - return rows - - -def _providers(model=MODEL_AUTO): - """Auto returns the permitted ladder; an explicit choice returns at most one provider.""" - import harness.analyst as analyst - - requested = str(model or MODEL_AUTO).strip().lower() - by_name = {p["name"]: p for p in analyst.PROVIDERS} - # ⚠ THE RECONCILIATION, at the ONE point the platform ladder is already in memory, so it costs - # nothing. A provider added to `analyst.PROVIDERS` would otherwise simply never appear here, - # and an env name changed there would make Query's availability answer quietly wrong. Both are - # invisible failures; this turns them into a row a reader can see [[limit-with-no-enforcer]]. - _LADDER_DRIFT[:] = sorted( - set(by_name) - set(QUERY_PROVIDER_ENV) - | {name for name, row in by_name.items() - if name in QUERY_PROVIDER_ENV and row.get("env") != QUERY_PROVIDER_ENV[name]}) - names = list(QUERY_PROVIDER_ORDER) if requested == MODEL_AUTO else [requested] - return [by_name[name] for name in names - if name in by_name and os.environ.get(by_name[name]["env"])] - - -def _history(state, thread_id): - """The prior turns of THIS thread, oldest first, in the transport's own message shape. - - ⛔ SORTED BY `(createdAt, role)`, NOT BY `createdAt` ALONE. `_submit` stamps the user turn and - the assistant turn with the SAME `now`, so on timestamp alone the pair ties and their order is - whatever the store's dict iteration happens to give — which is insertion order today and is - not a guarantee anybody wrote down. A replayed conversation with the answers before the - questions is worse than no replay: it reads as coherent and is backwards. - - ⚠ Scoped to one thread by construction. Replaying another thread's turns would leak one chat's - subject into another's answer, which is the kind of wrong that looks like a good answer. - """ - thread_id = str(thread_id or "") - if not thread_id: - return [] - rows = [row for row in state["messages"].values() - if isinstance(row, dict) and str(row.get("threadId") or "") == thread_id] - rows.sort(key=lambda row: (str(row.get("createdAt") or ""), - 0 if row.get("role") == "user" else 1)) - turns = [] - for row in rows[-MAX_HISTORY_TURNS:]: - content = " ".join(str(row.get("content") or "").split()) - if not content: - continue - role = "assistant" if row.get("role") == "assistant" else "user" - turns.append({"role": role, "content": content}) - # ⭐⭐ R20's "self-improving", and it is the ONLY thing that makes a rating more than a - # flag shipped without its writer. A thumbs-down becomes what it actually was: a turn in - # which the reader said the answer was not useful. Modelled as a USER turn rather than by - # editing the assistant's own words, because that is what happened, and because rewriting - # a stored answer to steer the next one is how a transcript stops being a record. - if role == "assistant" and row.get("rating") == "down": - reason = " ".join(str(row.get("ratingReason") or "").split())[:MAX_RATING_REASON] - turns.append({"role": "user", - "content": f"That answer was not helpful. {reason}".strip()}) - kept, spent = [], 0 - for turn in reversed(turns): - spent += len(turn["content"]) - if spent > MAX_HISTORY_CHARS and kept: - break - kept.append(turn) - kept.reverse() - return kept - - -def _spec_schema(field_keys): - col = {"type": "string", "enum": sorted(field_keys)} - return { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": [*QUERY_KINDS, "refused"]}, - "name": {"type": "string"}, - "refusal": {"type": "string"}, - "visible": {"type": "array", "items": col}, - # ⭐⭐ 2026-08-15 (owner: *"if you can build the view, the AI assistant should also be - # able to do it by using our tools on the backend"*). `rhs` and `important` were the - # only two things a person could express through `view_upsert` and this tool could not. - # - # ⛔ WITHOUT `rhs` THE ASSISTANT CANNOT STATE AN ERROR-CATCHER AT ALL — the one shape - # item 8a named by hand (*"price and COGS not matching"*). Every one of the 20 - # `FILTER_OPS` was already reachable, because they all read `value`; comparing a column - # against ANOTHER COLUMN is a different member (`aios_grid._clean_rhs`, CG-9) and it was - # simply absent here, so the model had no way to ask for it and no way to be told why. - # `kind` is `field` ONLY: `measure` needs a window and `stat` needs the population - # vocabulary, neither of which this snapshot-shaped reader carries — offering them - # would be a control that lies, which is the same rule the source chips follow. - "filters": {"type": "array", "items": {"type": "object", "properties": { - "colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)}, - "value": {"type": "string"}, - "rhs": {"type": "object", "properties": { - "kind": {"type": "string", "enum": ["field"]}, "colId": col, - }, "required": ["kind", "colId"]}, - }, "required": ["colId", "op"]}}, - # A personal legibility mark, exactly as `grid_events.view_upsert` treats it (wave 32 - # R5/C4) — not a lock, and no second permission wall. - "important": {"type": "boolean"}, - "filterConj": {"type": "string", "enum": ["and", "or"]}, - "sorts": {"type": "array", "items": {"type": "object", "properties": { - "colId": col, "dir": {"type": "string", "enum": ["asc", "desc"]}, - }, "required": ["colId", "dir"]}}, - "groupBy": col, - "aggregation": {"type": "object", "properties": { - "op": {"type": "string", "enum": ["count", "sum", "avg", "min", "max"]}, - "field": col, - }, "required": ["op"]}, - "stackField": col, - "dateField": col, - "colorField": col, - "sizeField": col, - }, - "required": ["kind"], - } - - -def _system_prompt(snapshot): - fields = snapshot["fields"] - cols = json.dumps([{"key": field["key"], "label": field.get("label", field["key"]), - "type": field.get("type", "text")} for field in fields], separators=(",", ":")) - return f"""You are the assistant inside this product, talking with one person about exactly one -database they already have permission to read. Reply in plain prose, and keep it short. - -DATABASE: {snapshot['database']} -SNAPSHOT VERSION: {json.dumps(snapshot['source_version'], default=str)} -PERMISSION-FILTERED RECORD COUNT: {len(snapshot['records'])} -FIELDS: {cols} -VIEW KINDS: {", ".join(QUERY_KINDS)} - -You may answer in words alone, and you may ask ONE short question back when the request is -ambiguous: which field is meant, which period, whether they want every record or a narrower set. -Prefer asking over guessing whenever the answer would change what you build. Earlier turns of this -conversation are above; use them, and read a pronoun as referring to what was just discussed. - -Call build_view ONLY when the person wants to SEE records: a table, a chart, a board, a list. Do -not call it to explain something, to confirm something, or to ask your question. When you do call -it you may also write one sentence saying what you built. - -Inside build_view: choose visible fields, supported filters and an optional aggregation. Use count -for record counts; sum, avg, min and max require one numeric field. The server computes and cites -every number from this exact snapshot. Never write SQL, invent fields, or name another database. -Return kind=refused with one plain sentence if the database cannot answer. - -To compare one column against ANOTHER column rather than a typed value, give the filter an rhs of -{{"kind":"field","colId":""}} and omit value. That is how you express questions like -"priced below what it costs us". Both columns must be in the FIELDS list above. -Set important=true when the view is one somebody should be chased about: an error, a mismatch, or -money at risk. Leave it out otherwise. - -Write with ordinary punctuation. Never use an em dash or an en dash: use a comma, a colon or a full -stop instead.""" - - -_FAILED_GEN = re.compile(r"(\{.*?\})\s*", re.S) - - -def _spec_from_400(body): - try: - value = ((json.loads(body) or {}).get("error") or {}).get("failed_generation") or "" - raw = (_FAILED_GEN.search(str(value)) or [str(value).strip()])[1] - result = json.loads(raw) - return result if isinstance(result, dict) else None - except Exception: - return None - - -# ⚠ BUILT FROM `chr`, AND A UNICODE ESCAPE IS NOT ENOUGH. `web_prose` reads a Python -# string's VALUE off the AST, not its spelling in the source, so an escape and the character -# itself are the SAME finding to it. Composing the class at runtime means no string literal in -# this file holds a dash, which is true rather than merely quiet: the only dashes in this module -# are the ones being removed. -_DASH = "[" + chr(0x2014) + chr(0x2013) + "]" - - -def _no_dashes(text): - """R6 applied where it is the ONLY place it can be applied: the model's own words. - - ⛔⛔ THE PROMPT INSTRUCTION IS NOT ENFORCEMENT, AND THIS IS MEASURED, NOT ANTICIPATED. The - system prompt ends with *"Never use an em dash or an en dash"* and the very next live turn came - back with *"Which view type would you like—grid, chart, list, or another?"* (cerebras, - 2026-08-16). Model prose reaches the screen exactly as a string literal does, and `web_prose` - scans SOURCE, so it cannot see a dash that arrives at runtime: the sweep every lane is doing - this wave is undone by our own assistant unless it is undone here. - - ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE. "10–20" means "10 to 20"; rewriting it as - "10, 20" states two numbers where the model stated a span, which is a wrong answer rather than - a punctuation fix. It gets its own rule, first. - """ - text = str(text or "") - text = re.sub(rf"(?<=\d)\s*{_DASH}\s*(?=\d)", " to ", text) - text = re.sub(rf"\s*{_DASH}\s*(?=[,.;:!?])", "", text) # abutting punctuation: it just goes - text = re.sub(rf"(?<=[,;:])\s*{_DASH}\s*", " ", text) # already punctuated: one space - return re.sub(rf"\s*{_DASH}\s*", ", ", text) - - -def _said(raw): - """The model's own words: one line of whitespace, no dash, and empty means nothing was said.""" - return _no_dashes(" ".join(str(raw or "").split())).strip() or None - - -def _from_chat(answer, provider): - """An injected transport may answer with a SPEC (a dict) or with PROSE (a string). - - ⛔ THE BRANCH BELONGS HERE, NOT AT THE CALLER. `_validate`'s first line refuses anything that - is not a dict, with "the assistant did not answer with a view" — so a prose answer passed - straight through would arrive as a REFUSAL, styled as one, and every conversation check would - pass against the wrong path while looking green [[one-question-two-normalizers]]. - """ - if isinstance(answer, str): - return None, _said(answer), provider, None - return answer, None, provider, None - - -def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None): - """Return ``(spec, said, provider, reason)`` without any source-data fallback. - - ``said`` is what the assistant SAID: the whole answer when it did not build a view, and the - sentence beside the view when it built one and talked as well. ``reason`` is set only when - something went wrong, so a prose ANSWER and a transport FAILURE are distinguishable one layer - up rather than both arriving as a bare sentence. - - ⭐⭐ W35-T31 · CONTRACT C7 (R9) — ``session`` IS HERE ONLY SO THE METER CAN BE TOLD WHOSE CALL - THIS WAS, and PRD amendment A1 is why it is an argument rather than ambient state: a - ``ContextVar`` bound in ``deps.require_session`` reads back ``None`` inside this function on the - SAME thread, so a ledger built on one would have recorded nothing and shown a dashboard of - zeros indistinguishable from a quiet week. - - ⚠ OPTIONAL, DELIBERATELY. ``verify_query`` calls this function directly with an injected - ``chat`` and no session, and a required argument would have made every one of those calls a - signature change. An omitted session is COUNTED in ``usage_ledger.UNATTRIBUTED`` and reported - to an admin, never dropped. - """ - requested = str(model or MODEL_AUTO).strip().lower() - if requested not in model_choices(): - return None, f"the selected model ({requested or model}) is unavailable", None, "model_unavailable" - tools = [{"type": "function", "function": { - "name": "build_view", "description": "Emit a virtual-view spec or a refusal.", - "parameters": _spec_schema([field["key"] for field in snapshot["fields"]]), - }}] - messages = [{"role": "system", "content": _system_prompt(snapshot)}, - *(history or []), - {"role": "user", "content": question}] - if chat is not None: - provider = requested if requested != MODEL_AUTO else "injected" - return _from_chat(chat(messages, tools), provider) - - providers = _providers(requested) - if not providers: - sentence = (f"the selected model ({requested}) is unavailable" if requested != MODEL_AUTO - else "the assistant is not configured on this deployment") - return None, sentence, None, "model_unavailable" if requested != MODEL_AUTO else "not_configured" - - import requests - - import usage_ledger - - def _book(provider, body): - """⭐⭐ C7 — ONE LEDGER LINE PER PROVIDER RESPONSE THIS FUNCTION READS. - - ⛔ HERE, INSIDE THE LADDER, AND NOT AT THE RETURN. The ladder tries providers in order and - a failed one has already SPENT tokens at that vendor; booking only the winner would report - a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads - gets a line, including the one whose answer turned out to be empty. - ⚠ `tokens_from` returns `(None, None)` when the provider did not say, and `None` is carried - through rather than coerced to 0 — a call booked at zero is an unmeasured call presented as - a free one. `usage_ledger` counts it as UNMEASURED so the total reads as a floor. - ⚠ `record` never raises, by its own contract, so this cannot break the assistant. - """ - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record( - "assistant", provider["name"], provider["model"], ins, outs, - total=usage_ledger.total_from(body), - st=getattr(session, "runtime", None), user=getattr(session, "uname", "")) - - last = None - for provider in providers: - try: - response = requests.post( - provider["url"], timeout=60, - headers={"Authorization": f"Bearer {os.environ[provider['env']]}"}, - # ⛔⛔ `auto`, NOT `required`, AND THAT ONE WORD IS R16. Under `required` the model - # had to emit a `build_view` call for EVERY turn: it could not answer a question, - # could not ask one back, and could not decline to build. "The AI chat must TALK - # with the user, not only build queries, and decide for itself whether to create a - # query view" is unreachable with the tool forced, whatever the prompt says. - # ⚠ AND IT IS A LIVE VENDOR-BEHAVIOUR CHANGE THAT NO GATE HERE CAN SEE, because - # `verify_query` injects `chat` and never builds this body. `_spec_from_400` below - # exists precisely because providers differ in how they emit a forced call; under - # `auto` a provider may also decline to build for a view-shaped question. That is - # why this ticket's evidence is a real three-turn run against a real provider and - # not a green gate. - json={"model": provider["model"], "messages": messages, "tools": tools, - "tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200}, - ) - except Exception as exc: - last = f"{provider['name']}: {type(exc).__name__}" - continue - if response.status_code == 400: - spec = _spec_from_400(response.text) - if spec is not None: - return spec, None, provider["name"], None - last = f"{provider['name']}: 400" - continue - if response.status_code != 200: - last = f"{provider['name']}: HTTP {response.status_code}" - continue - try: - body = response.json() - # C7: booked from the RAW body, before anything below can raise on its shape. A - # provider that answered 200 and billed for it has spent tokens whether or not this - # function can read what it said. - _book(provider, body) - answer = body["choices"][0]["message"] - said = _said(answer.get("content")) - calls = answer.get("tool_calls") or [] - if calls: - # The prose rides ALONG with the view when the model wrote both, so the reader gets - # a sentence instead of `_explain`'s machine description of its own output. - spec = json.loads(calls[0]["function"].get("arguments") or "{}") - return spec, said, provider["name"], None - if said: - return None, said, provider["name"], None - # ⚠ Neither a call nor a word is a FAILED turn, not a silent one: falling through to - # the next provider is right, and swallowing it as an empty answer would show the - # reader a blank reply [[empty-answer-vs-unfinished-answer]]. - last = f"{provider['name']}: an empty answer" - except Exception as exc: - last = f"{provider['name']}: unreadable answer ({type(exc).__name__})" - if requested != MODEL_AUTO: - return None, f"the selected model ({requested}) is unavailable", None, "model_unavailable" - return (None, "the assistant could not be reached just now (" + (last or "no provider") + ")", - None, "provider_unreachable") - - -def _validate(spec, fields): - """Return a cleaned, source-independent virtual-view config or a named refusal.""" - if not isinstance(spec, dict): - return None, "the assistant did not answer with a view", "no_spec" - by_key = {str(field.get("key")): str(field.get("type") or "text") for field in fields} - keys = set(by_key) - kind = spec.get("kind") - if kind == "refused": - # Model-authored, so it faces the same R6 wall the model's chat prose does. - return None, _said(spec.get("refusal")) or "this database cannot answer that question", "model_refused" - if kind in QUERY_EXCLUDED or kind not in QUERY_KINDS: - return None, "that kind of view cannot be built from this question", "unsupported_kind" - - named = set(spec.get("visible") or ()) - for item in spec.get("filters") or (): - if isinstance(item, dict) and item.get("colId"): - named.add(str(item["colId"])) - # ⛔ THE RIGHT-HAND COLUMN IS A COLUMN AND MUST FACE THE SAME `missing` CHECK. Collecting - # only the left side is what makes D-229 possible one layer down: `clean_filter_tree` does - # NOT drop a leaf whose field-rhs names a column that does not exist — it keeps the leaf, - # strips the `rhs`, blanks the value, and `filter_sql.is_rule_active` then reports the rule - # INACTIVE. An inactive rule narrows nothing, so "margin under 10%" would come back as a - # view listing the ENTIRE catalogue under an error-catcher's name, with nothing red. - # Naming it here turns that into the ordinary "this database does not have: X" refusal. - rhs = item.get("rhs") if isinstance(item, dict) else None - if isinstance(rhs, dict) and rhs.get("colId"): - named.add(str(rhs["colId"])) - for item in spec.get("sorts") or (): - if isinstance(item, dict) and item.get("colId"): - named.add(str(item["colId"])) - for key in ("groupBy", "stackField", "dateField", "colorField", "sizeField"): - if spec.get(key): - named.add(str(spec[key])) - raw_aggregation = spec.get("aggregation") or {"op": "count"} - if isinstance(raw_aggregation, dict) and raw_aggregation.get("field"): - named.add(str(raw_aggregation["field"])) - missing = sorted(named - keys) - if missing: - return None, "this database does not have: " + ", ".join(missing), "unknown_columns" - - visible = [str(key) for key in (spec.get("visible") or ()) if str(key) in keys][:MAX_VISIBLE] - if not visible: - return None, "that question did not name any fields to show", "no_columns" - raw_filters = [item for item in (spec.get("filters") or ()) if isinstance(item, dict)] - filters = _grid().clean_filter_tree(raw_filters, keys) - if len(filters) != len(raw_filters): - return None, "part of that filter is unsupported", "filter_dropped" - # ⛔⛔ A SECOND, NARROWER CHECK, AND THE LENGTH CHECK ABOVE CANNOT DO ITS JOB (D-229). - # A dropped leaf changes the COUNT; a stripped `rhs` does not — the leaf survives, so - # `len(filters) == len(raw_filters)` and the refusal above never fires. The failure is - # therefore silent in exactly the direction that matters: the condition stops narrowing and - # the view answers with every record. Assert the member survived, per leaf. - # ⚠ The `named` pass above already refuses an rhs naming a column this database lacks, so - # reaching here means something ELSE stripped it (a type the comparand cannot take, a future - # `_clean_rhs` rule). Both doors, because the two catch different causes and the cost of - # missing this one is a wrong answer that looks right. - for sent, kept in zip(raw_filters, filters): - if sent.get("rhs") and not kept.get("rhs"): - return None, "that column cannot be compared against another column", "rhs_dropped" - - aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {} - op = str(aggregation.get("op") or "").lower() - field = aggregation.get("field") - if op not in {"count", "sum", "avg", "min", "max"}: - return None, "the assistant gave an unsupported aggregation", "bad_aggregation" - if op == "count": - field = None - elif field not in keys or by_key.get(field) not in {"int", "currency", "pct"}: - return None, "that aggregation needs one visible numeric field", "bad_aggregation" - - display = {"mode": kind} - for ref, families, required in _MODE_REFS.get(kind, ()): - value = spec.get(ref) - if required and not value: - return None, f"a {kind} view needs {ref}", "missing_ref" - if value and by_key.get(value) not in families: - return None, f"{ref} has the wrong field type", "wrong_ref_type" - if value: - display[ref] = value - cleaned_display = _grid()._clean_display(display, keys) if kind != "grid" else None - if kind != "grid" and not cleaned_display: - return None, f"this product could not build a {kind} view", "display_dropped" - - return { - "kind": kind, - # The view NAME is model-authored too, and it is the string that ends up in the rail, in - # the flyout and on the artefact card. R6 reaches it here or nowhere. - "name": (_said(spec.get("name")) or "Query")[:60], - "visible": visible, - "filters": filters, - "filterConj": "or" if spec.get("filterConj") == "or" else "and", - "sorts": [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} - for item in (spec.get("sorts") or []) if isinstance(item, dict) - and item.get("colId") in keys][:3], - "groupBy": spec.get("groupBy") if spec.get("groupBy") in keys else None, - "aggregation": {"op": op, "field": field}, - "display": cleaned_display, - # ⚠ `is True`, not truthy, and UNCONDITIONAL — the same two rules `grid_events.view_upsert` - # follows for this key. `is True` so a model emitting the string "false" does not mark a - # view; unconditional so the mark is REMOVABLE rather than a flag that can be set and never - # cleared (a key written only when present leaves a stored `true` alive forever). - "important": spec.get("important") is True, - }, None, None - - -def _explain(view, fields): - label = {field["key"]: str(field.get("label") or field["key"]) for field in fields} - visible = ", ".join(label.get(key, key) for key in view["visible"][:6]) - result = f"{view['kind']} view of {visible}" - if view["filters"]: - result += "; filtered records only" - if view.get("groupBy"): - result += f"; grouped by {label.get(view['groupBy'], view['groupBy'])}" - agg = view["aggregation"] - if agg["op"] != "count": - result += f"; {agg['op']} of {label.get(agg['field'], agg['field'])}" - return result + "." - - -def _said_fallback(artifact): - """What a build turn SAYS when the provider volunteered no sentence of its own. - - ⚠ MEASURED, 2026-08-16, cerebras: a view-shaped question comes back as a tool call with - `content: null`. So this is the sentence a person reads on MOST build turns, not a rare - fallback, and R16 ("the AI chat must TALK with the user") is decided here rather than in the - prompt. `_explain` used to be it, and it is a receipt for our own output: *"grid view of - Company, Owner, Deal value; filtered records only."* It keeps its real job ON THE ARTEFACT, - where it labels the thing it describes. - """ - numeric = artifact.get("numeric") or {} - value = numeric.get("value") - if value is None: - return f"I built {artifact['name']}." - if isinstance(value, float) and value.is_integer(): - value = int(value) - number = f"{value:,}" if isinstance(value, (int, float)) else str(value) - label = " ".join(str(numeric.get("label") or "Records").split()) - return f"I built {artifact['name']}. {label}: {number}." - - -def _numeric_result(view, records): - aggregation = view["aggregation"] - if aggregation["op"] == "count": - return {"label": "Matching records", "value": len(records), "contributing_record_count": len(records)} - values = [] - for record in records: - try: - value = record.get(aggregation["field"]) - if value is not None and not isinstance(value, bool): - values.append(float(value)) - except (TypeError, ValueError): - continue - if not values: - return {"label": aggregation["op"], "value": None, "contributing_record_count": 0} - op = aggregation["op"] - value = {"sum": sum(values), "avg": sum(values) / len(values), "min": min(values), "max": max(values)}[op] - return {"label": f"{op.title()} of {aggregation['field']}", "value": value, - "contributing_record_count": len(values)} - - -def _referenced_fields(view): - """The citation names every source field that affected the displayed result.""" - out = list(view.get("visible") or ()) - for node in view.get("filters") or (): - if isinstance(node, dict) and node.get("colId"): - out.append(str(node["colId"])) - for node in view.get("sorts") or (): - if isinstance(node, dict) and node.get("colId"): - out.append(str(node["colId"])) - for key in ("groupBy",): - if view.get(key): - out.append(str(view[key])) - aggregation = view.get("aggregation") or {} - if aggregation.get("field"): - out.append(str(aggregation["field"])) - return list(dict.fromkeys(out)) - - -def _effective_filters(snapshot, view): - """Keep the source request and generated-view predicates distinct in provenance.""" - return { - "source": _safe(snapshot.get("filters")), - "view": {"conj": view.get("filterConj", "and"), - "nodes": _safe(view.get("filters") or [])}, - } - - -def _view_records(snapshot, view): - """Apply the exact validated virtual-view filter before calculating a cited number.""" - nodes = view.get("filters") or [] - if not nodes: - return list(snapshot["records"]) - from harness import filter_eval - tree = {"conj": view.get("filterConj", "and"), "nodes": nodes} - return [row for row in snapshot["records"] - if filter_eval.matches(tree, row, snapshot["fields"])] - - -def _citation(citation_id, snapshot, view, numeric, view_id): - return { - "id": citation_id, - "href": f"#/query?view={view_id}&citation={citation_id}", - "database": snapshot["database"], - "snapshot": {"kind": snapshot["source_kind"], "version": _safe(snapshot["source_version"])}, - "fields": [field["key"] for field in snapshot["fields"] - if field.get("key") in set(_referenced_fields(view))], - "filters": _effective_filters(snapshot, view), - "permission_scope_applied": snapshot["permission_scope_applied"], - "aggregation": _safe(view["aggregation"]), - "contributing_record_count": numeric["contributing_record_count"], - "retrieved_at": snapshot["retrieved_at"], - } - - -def _citation_complete(citation): - """A numeric result is not publishable without complete provenance.""" - required = {"database", "snapshot", "fields", "filters", "aggregation", - "contributing_record_count", "retrieved_at", "href"} - snapshot = citation.get("snapshot") if isinstance(citation, dict) else None - return (isinstance(citation, dict) and required <= set(citation) - and bool(citation["database"]) and isinstance(snapshot, dict) - and "version" in snapshot and bool(citation["retrieved_at"]) - and isinstance(citation["fields"], list) - and isinstance(citation["aggregation"], dict)) - - -def _public_view(view): - source = view["source"] - # ⭐⭐ W35-T25 · CONTRACT C4 — `edited` IS DERIVED, NEVER STORED, and that is deliberate. - # A stored boolean beside two specs is a third source of truth that can disagree with both; - # the only honest answer to "has this been changed" is "compare it". It also means a Revert - # that restores the spec clears the badge by construction rather than by remembering to. - original = view.get("original_spec") - return { - "id": view["id"], "viewId": view["id"], "scope": source["database"], - "name": view["name"], "description": str(view.get("description") or ""), - "kind": view["view"]["kind"], "question": view["question"], - "explain": view["explain"], "threadId": view["threadId"], "createdAt": view["createdAt"], - "virtual": True, "source": _safe(source), "view": _safe(view["view"]), - "citationIds": list(view["citationIds"]), "numeric": _safe(view["numeric"]), - # ⚠ AN ARTEFACT MADE BEFORE THIS WAVE HAS NO ORIGINAL, so it reports `edited: false` and - # sends no `original_spec` — and the client shows neither the badge nor Revert. C4 is - # explicit that this is the right answer: a Revert with nothing to revert to is worse - # than an absent one, and it is the case an NC in `verify_query` covers. - "original_spec": _safe(original) if isinstance(original, dict) else None, - "edited": bool(isinstance(original, dict) and _safe(view["view"]) != _safe(original)), - } - - -def _source_still_permitted(session, source, permitted=None): - """A persisted artefact never outlives the caller's current data permission. - - ⚠ `permitted` is the BATCH answer from `deps.assistant_source_status` — one rows-free - resolution for every source at once, rather than one whole-document read per saved view. That - function's docstring carries the measurement. The single-source path below stays for callers - holding exactly one artefact (the workspace-event door), and asks the same helper. - - ⛔ `permitted`, NEVER `answerable`. A source whose rows are served through the connector mirror - cannot be ASKED and can still be SEEN: the artefact was built from a snapshot that was legal - when it was taken, and hiding it because the reader can no longer make a NEW one would read as - deletion. The two verdicts are separate for that reason. - """ - database = (source or {}).get("database") if isinstance(source, dict) else None - if not database: - return False - if permitted is None: - permitted = {key for key, row in assistant_source_status(session, [database]).items() - if row["permitted"]} - return database in permitted - - -def _public_state(state, session): - # ONE rows-free batch answers both questions: which sources this caller may still see (which - # artefacts stay listed) and which of them can actually be asked (which chips are live). - status = assistant_source_status(session) - for row in state["views"].values(): - key = (row.get("source") or {}).get("database") if isinstance(row.get("source"), dict) else None - if key and key not in status: - # An artefact whose source is no longer enumerable still gets a verdict rather than a - # KeyError — it resolves to "not permitted" and the artefact drops out, which is the - # same answer the per-view wall gave. - status.update(assistant_source_status(session, [key])) - permitted = {key for key, row in status.items() if row["permitted"]} - allowed_views = [row for row in state["views"].values() - if _source_still_permitted(session, row.get("source"), permitted)] - allowed_ids = {row["id"] for row in allowed_views} - allowed_citations = {citation_id for row in allowed_views - for citation_id in row.get("citationIds") or []} - threads = sorted(state["threads"].values(), key=lambda row: row.get("updatedAt", ""), reverse=True) - messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", "")) - views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True) - messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids] - return {"threads": _safe(threads), "messages": _safe(messages), "views": views, - "citations": _safe([row for key, row in state["citations"].items() - if key in allowed_citations]), "models": model_choices(), - # The model picker's own data, the same shape `sources` uses for databases. - "modelStatus": model_status(), - # The chip row's own data: a source this caller holds but cannot ask, and WHY. - # ⭐⭐ D-276 — `or row.get("visible")`. A source the caller can open elsewhere in the - # product but the assistant cannot read is now LISTED with its cause, instead of - # vanishing from a picker that shows every other database they hold. It arrives with - # `answerable: False` and a `reason`, which is the SAME shape the mirror-served grids - # already use, so the chip row needs no new state to render it. - # ⛔ `permitted` STILL GATES ARTEFACTS — `allowed_views` above is unchanged. This widens - # what is DESCRIBED, never what can be read or kept. - "sources": _safe([{"database": key, "answerable": row["answerable"], - "reason": row["reason"]} - for key, row in sorted(status.items()) - if row["permitted"] or row.get("visible")])} - - -@router.get("/query") -def list_queries(session: Session = Depends(require_session)): - return _public_state(_state(session), session) - - -# ⭐⭐ W35-T25 · CONTRACT C4 (owner item 4 / R3) — WHICH SPEC MEMBERS AN EDIT MAY MOVE. -# -# ⛔ AN ALLOW-LIST, AND IT IS THE SECURITY BOUNDARY OF THIS WHOLE TICKET. The client sends a -# `SavedView`, a shape it composes itself, and merging it wholesale would let a caller rewrite -# `kind` (which decides the renderer and the citation's own claim), `aggregation` (the number this -# artefact CITED) or `name`. R3 opens the view SPEC: *"filters, sort, group, visible columns, -# widths"*, plus the row height and column order that carry them. Nothing else. -# -# ⚠ `aggregation` IS DELIBERATELY ABSENT even though a person can change it on an ordinary grid. -# The artefact's citation records `contributing_record_count` and an op computed from THIS -# aggregation; letting an edit move it would leave a cited number describing a calculation the -# artefact no longer performs, which is the one thing every provenance rule in this module exists -# to prevent. -# -# ⚠ A COMMENT, NOT A BARE MODULE-LEVEL STRING. `verify_prose` reads a free-floating string literal -# as candidate copy, so writing this as a `"""..."""` above the constant put an em dash into the -# gate's own census as a 296th finding. A `#` block is out of scope by construction. -QUERY_EDITABLE_SPEC = ("visible", "order", "widths", "filters", "filterConj", "sorts", - "groupBy", "rowHeightMode", "frozenCount", "colorBy", "display") - - -def _clean_spec_edit(config, spec, fields): - """Merge a client view config onto the artefact's spec, cleaned against ITS OWN fields. - - ⛔ THE FIELD KEYS COME FROM THE STORED SOURCE, never from the request. The artefact carries - the snapshot's field list, so this needs no database read and cannot be widened by a caller - naming a column the snapshot did not have. - """ - keys = {str(field.get("key")) for field in (fields or []) if isinstance(field, dict)} - out = copy.deepcopy(spec) if isinstance(spec, dict) else {} - if not isinstance(config, dict): - return out - for member in QUERY_EDITABLE_SPEC: - if member not in config: - continue - value = config[member] - if member in ("visible", "order"): - cleaned = [str(key) for key in value if str(key) in keys] if isinstance(value, list) else [] - # ⚠ An EMPTY visible list is refused rather than stored: `_validate` already treats - # "no columns" as a refusal at creation, and a grid showing nothing is not an edit - # somebody meant to make. - if member == "visible" and not cleaned: - continue - out[member] = cleaned[:MAX_VISIBLE] - elif member == "widths": - out[member] = {str(key): int(width) for key, width in value.items() - if str(key) in keys and isinstance(width, (int, float)) - and 0 < float(width) <= 2000} if isinstance(value, dict) else {} - elif member == "filters": - out[member] = _grid().clean_filter_tree( - [item for item in value if isinstance(item, dict)], keys) if isinstance(value, list) else [] - elif member == "filterConj": - out[member] = "or" if value == "or" else "and" - elif member == "sorts": - out[member] = [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} - for item in value if isinstance(item, dict) and item.get("colId") in keys][:3] \ - if isinstance(value, list) else [] - elif member in ("groupBy", "colorBy"): - out[member] = value if value in keys else None - elif member == "rowHeightMode": - out[member] = value if value in ("short", "medium", "tall", "extra") else None - elif member == "frozenCount": - out[member] = max(0, min(6, int(value))) if isinstance(value, (int, float)) else 0 - elif member == "display": - out[member] = _grid()._clean_display(value, keys) if isinstance(value, dict) else out.get("display") - return out - - -@router.post("/query/{qid}/events") -def mutate_query_workspace(qid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """The only grid-mutation transport for a virtual Query workspace. - - ⭐⭐ W35-T25 (owner item 4 / R2) — `view_upsert` ON THE ARTEFACT ITSELF IS NOW ACCEPTED. It - used to 409 `query_workspace_immutable` for everything but a delete, on the reading that an AI - artefact is a snapshot. R2 replaces that reading: the SPEC is the reader's to shape, the - CITATION and the numbers behind it are not. `QUERY_EDITABLE_SPEC` above is where that line is - drawn, and `original_spec` is what makes the change undoable. - - Creating a SECOND view inside the workspace is still refused — an artefact holds exactly one — - and deleting the caller's personal artefact still removes it. The binding key, not a - client-supplied source scope, identifies it. - """ - qid = str(qid) - body = body if isinstance(body, dict) else {} - binding = body.get("workspaceBinding") - event = body.get("event") - if not (isinstance(binding, dict) and binding.get("kind") == "query" - and str(binding.get("key") or "") == qid): - raise err(400, "query_workspace_mismatch", - "the Query workspace binding must name this virtual artefact") - if not isinstance(event, dict) or str(event.get("type") or "") not in QUERY_WORKSPACE_EVENTS: - raise err(400, "unsupported_query_workspace_event", - "Query accepts only view_create, view_upsert, or view_delete events") - - state = _state(session) - view = state["views"].get(qid) - if view is None or not _source_still_permitted(session, view.get("source")): - # A caller cannot use an opaque Query key to learn about a revoked artefact. - raise err(404, "unknown_query", "that Query artefact does not exist") - - event_type = str(event["type"]) - if event_type == "view_upsert": - sent = event.get("view") - sent = sent if isinstance(sent, dict) else {} - # ⛔ THE ID IS CHECKED THE SAME WAY THE DELETE'S IS. An upsert naming a different view is - # a create wearing an update's name, and a Query workspace holds exactly one view. - if str(sent.get("id") or "") != qid: - raise err(400, "query_workspace_mismatch", - "a Query view edit must name the same virtual artefact as its binding") - spec = _clean_spec_edit(sent.get("config"), view.get("view"), (view.get("source") or {}).get("fields")) - - def apply_edit(raw): - current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() - row = (current.get("views") or {}).get(qid) - if not isinstance(row, dict): - return current - # ⚠ BACKFILLED HERE, and only when absent: an artefact created before this wave has no - # original, and the FIRST edit is the last moment its pre-edit spec still exists. Not - # backfilling would leave it permanently unrevertable; backfilling unconditionally - # would make Revert restore the latest edit. - if not isinstance(row.get("original_spec"), dict): - row["original_spec"] = _safe(row.get("view")) - row["view"] = _safe(spec) - current["views"][qid] = row - return current - - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was saved") - session.runtime.update(_namespace_key(session), apply_edit, flush="async") - return {"workspaceBinding": {"kind": "query", "key": qid}, "event": event_type, - "view": _public_view(apply_edit(_state(session))["views"][qid])} - - if event_type != "view_delete": - raise err(409, "query_workspace_immutable", - "a Query workspace holds one view, so it cannot take another") - if str(event.get("viewId") or "") != qid: - raise err(400, "query_workspace_mismatch", - "a Query delete must name the same virtual artefact as its binding") - - deleted = delete_query(qid, session) - return {"workspaceBinding": {"kind": "query", "key": qid}, - "event": event_type, **deleted} - - -@router.post("/query/{qid}/revert") -def revert_query(qid: str, session: Session = Depends(require_session)): - """⭐⭐ CONTRACT C4 (R3) — restore the AI's original SPEC, and only the spec. - - ⛔ WHAT THIS DOES NOT DO, which the client's confirm says out loud BEFORE it acts: it does not - undo anything the reader changed in the SOURCE database. A Query view is live now, so a cell - edit made through it is a real write to a real record — reverting a view's filters cannot and - must not walk those back. R3 is explicit that the dialog states this before it acts, because a - Revert that silently leaves data changed is worse than one that never offered. - - ⚠ 409, not 404, when there is no original: the artefact exists and is readable, and the caller - asked for something that does not exist FOR IT. A 404 would say the artefact is gone. - """ - qid = str(qid) - # ⚠ `_artifact_or_404` returns `(state, row)`, not the row. Read it as a pair. - _state_now, view = _artifact_or_404(session, qid) - original = view.get("original_spec") - if not isinstance(original, dict): - raise err(409, "no_original_spec", - "this view was created before the assistant kept an original, so there is " - "nothing to revert to") - - def restore(raw): - current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() - row = (current.get("views") or {}).get(qid) - if not isinstance(row, dict): - return current - row["view"] = _safe(row.get("original_spec")) - current["views"][qid] = row - return current - - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was reverted") - session.runtime.update(_namespace_key(session), restore, flush="async") - return _public_view(restore(_state(session))["views"][qid]) - - -def _sources(body, target, session): - raw = body.get("sources") if isinstance(body, dict) else None - sources = [str(item).strip() for item in raw] if isinstance(raw, list) else [] - sources = list(dict.fromkeys(item for item in sources if item)) - if target not in sources: - sources.insert(0, target) - # Source chips are a permissioned selection, not untrusted labels remembered in a thread. - # The model still receives ONLY `target`'s snapshot below; these calls do not pass a second - # source to it and D's helper refuses cache misses, mirrors and unshared databases. - for source in sources: - assistant_read_scope(session, source, fields=[], filters=None) - return sources - - -def _submit(body, session, chat=None): - body = body if isinstance(body, dict) else {} - question = " ".join(str(body.get("question") or "").split()) - target = str(body.get("database") or body.get("scope") or "").strip() - selected_model = str(body.get("model") or MODEL_AUTO).strip().lower() - if not question: - raise err(400, "bad_request", "no question was asked") - if len(question) > MAX_QUESTION: - raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters") - if not target: - raise err(400, "bad_request", "one target database must be selected") - if selected_model not in model_choices(): - raise err(400, "unknown_model", "that model is not available in Query") - - # This call happens before model selection and is the only permitted source-data read. - snapshot = assistant_read_scope(session, target, fields=None, filters=body.get("filters")) - sources = _sources(body, target, session) - state = _state(session) - thread_id = str(body.get("threadId") or "").strip() - if thread_id and thread_id not in state["threads"]: - raise err(404, "unknown_thread", "that chat does not exist") - if not thread_id: - thread_id = _new_id("thread") - now = _now_iso() - user_message_id = _new_id("message") - assistant_message_id = _new_id("message") - # R16: the turns already on screen go with the question. `state` was read above, BEFORE this - # turn's own messages exist, so the replay is strictly the prior conversation. - # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s - # docstring for why it is an argument and not ambient state (PRD amendment A1). - spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat, - history=_history(state, thread_id), session=session) - view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason) - - artifact = None - citation = None - if view is not None: - if len(state["views"]) >= MAX_ARTIFACTS: - raise err(400, "query_limit", f"you already have {len(state['views'])} Query artefacts") - view_id = _new_id("query") - citation_id = _new_id("citation") - numeric = _numeric_result(view, _view_records(snapshot, view)) - citation = _citation(citation_id, snapshot, view, numeric, view_id) - if not _citation_complete(citation): - raise RuntimeError("Query refused to persist an incomplete numeric citation") - source = {"database": snapshot["database"], "label": snapshot["label"], - "source_kind": snapshot["source_kind"], "source_version": _safe(snapshot["source_version"]), - "retrieved_at": snapshot["retrieved_at"], - "fields": _safe(snapshot["fields"]), "filters": _safe(snapshot["filters"]), - "permission_scope_applied": snapshot["permission_scope_applied"]} - artifact = {"id": view_id, "threadId": thread_id, "name": view["name"], "description": "", - "question": question, - "explain": _explain(view, snapshot["fields"]), "createdAt": now, "source": source, - "view": _safe(view), "citationIds": [citation_id], "numeric": numeric, - # ⭐⭐ W35-T25 · CONTRACT C4 (R3) — THE AI'S OWN SPEC, WRITTEN ONCE, HERE. - # R2 makes a Query view editable, so `view` moves from now on. This is the copy - # "Revert to AI original" restores, and the thing `edited` is measured against. - # ⛔ WRITTEN AT CREATE AND NOWHERE ELSE. Re-stamping it on any later write would - # make Revert restore the most recent edit — a Revert that reverts to nothing, - # which C4 names as worse than no Revert at all. - # ⚠ `_safe(view)` twice, not the same object twice: `view` is mutable and a - # shared reference would let an edit rewrite the original through the alias. - "original_spec": _safe(view), - "model": provider, "requestedModel": selected_model} - - assistant_message = { - "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now, - "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"), - "targetDatabase": target, "requestedModel": selected_model, "model": provider, - "reason": refusal_code, "viewId": artifact["id"] if artifact else None, - "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None, - } - user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now, - "content": question, "sources": sources, "targetDatabase": target, - "requestedModel": selected_model} - - def update(raw): - current = _blank_state() - if isinstance(raw, dict): - for key in ("threads", "messages", "citations", "views"): - if isinstance(raw.get(key), dict): - current[key] = copy.deepcopy(raw[key]) - thread = current["threads"].get(thread_id) or {"id": thread_id, "createdAt": now} - # ⚠ THE FIRST QUESTION NAMES THE CHAT, and R16 is what makes that matter. While every turn - # was a fresh one-shot the title could only be the last thing asked; now that a thread is a - # conversation, retitling it on every follow-up renames the history entry out from under - # the reader, and "and the other one?" is a useless name for anything. - thread.update({"updatedAt": now, "title": thread.get("title") or question[:80], "sources": sources, - "model": selected_model, "activeViewId": artifact["id"] if artifact else thread.get("activeViewId")}) - current["threads"][thread_id] = thread - current["messages"][user_message_id] = user_message - current["messages"][assistant_message_id] = assistant_message - if artifact: - current["views"][artifact["id"]] = artifact - current["citations"][citation["id"]] = citation - return current - - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable; no chat was saved") - session.runtime.update(_namespace_key(session), update, flush="async") - return {"thread": _safe(update(state)["threads"][thread_id]), "userMessage": _safe(user_message), - "message": _safe(assistant_message), - "view": _public_view(artifact) if artifact else None, "citations": [citation] if citation else []} - - -@router.post("/query/chat") -def submit_chat(body: dict = Body(default=None), session: Session = Depends(require_session)): - return _submit(body, session) - - -@router.delete("/query/threads/{tid}") -def delete_thread(tid: str, session: Session = Depends(require_session)): - """Remove one of the caller's own chats, and only the chat. - - ⛔ THE ARTEFACTS SURVIVE, AND THAT IS THE POINT. A Query view is a durable object in its own - right — it appears in Query's rail, other people's links can point at it, and the owner's - instruction was that AI views LIVE THERE rather than inside the conversation that happened to - produce them. Cascading the delete would make tidying up a chat silently destroy work. Views - are deleted from Query's own rail, one at a time, by `delete_query` below. - - ⚠ It exists because the history had no prune. `delete_query` removed a view and left its - thread, so the panel could only ever grow — a navigation you cannot manage is half a - navigation, and a person clearing test chats found that out first. - """ - tid = str(tid) - state = _state(session) - if tid not in state["threads"]: - raise err(404, "unknown_thread", "that chat does not exist") - - def update(raw): - current = _blank_state() - if isinstance(raw, dict): - for key in ("threads", "messages", "citations", "views"): - if isinstance(raw.get(key), dict): - current[key] = copy.deepcopy(raw[key]) - current["threads"].pop(tid, None) - for message_id in [key for key, row in current["messages"].items() - if (row or {}).get("threadId") == tid]: - current["messages"].pop(message_id, None) - # ⚠ THE VIEWS' `threadId` IS LEFT EXACTLY AS IT WAS, dangling. Blanking it looks tidier and - # is a data-loss bug: `queryApi.saved()` refuses a row with an empty `threadId`, so every - # artefact built in the deleted chat would silently vanish from Query's rail — the exact - # "tidying up destroys work" outcome this door is written not to have - # [[a-record-can-outlive-its-subject]]. Nothing resolves the id but the chat panel, which - # only ever looks up the thread it is showing. - return current - - session.runtime.update(_namespace_key(session), update, flush="async") - return {"deleted": tid} - - -@router.delete("/query/{qid}") -def delete_query(qid: str, session: Session = Depends(require_session)): - qid = str(qid) - state = _state(session) - if qid not in state["views"]: - raise err(404, "unknown_query", "that Query artefact does not exist") - - def update(raw): - current = _blank_state() - if isinstance(raw, dict): - for key in ("threads", "messages", "citations", "views"): - if isinstance(raw.get(key), dict): - current[key] = copy.deepcopy(raw[key]) - view = current["views"].pop(qid, None) - for citation_id in (view or {}).get("citationIds") or []: - current["citations"].pop(citation_id, None) - return current - - session.runtime.update(_namespace_key(session), update, flush="async") - return {"deleted": qid} - - -# ══ R20 — A QUERY VIEW HAS EVERY FUNCTION A DATABASE VIEW HAS ═══════════════════════════════════ -# -# ⛔⛔ THE DECISION THIS TICKET ASKED FOR, WRITTEN DOWN RATHER THAN IMPLIED, because it is the line -# every later reader will need and there is no other place it exists. -# -# **IMMUTABLE, and unchanged by R20:** the generated SPEC and its provenance. `view` (kind, visible, -# filters, sorts, aggregation, display, important), `source` (the snapshot, its version, its -# retrieval time, the permission scope), `question`, `citationIds`, `numeric`. A cited number is -# only worth citing if the thing it was computed from cannot be edited underneath it, so -# `mutate_query_workspace` still answers 409 `query_workspace_immutable` for `view_create` and -# `view_upsert`, and `QUERY_MUTATION_POLICY` on the client is untouched. -# -# **OPENED:** `name` and `description`. They are LABELS, not the answer: changing them cannot make -# a citation wrong. Plus a server-side DUPLICATE (the client never supplies a spec, so a copy is a -# copy) and an EXPORT, which is a read. -# -# ⛔ AND THE REASON THESE ARE NAMED DOORS RATHER THAN AN OPENED `view_upsert`, which is what the -# ticket's `how:` first suggested: the client refuses `view_upsert` on a Query binding LOCALLY, -# before any request leaves the browser (`queryPreview.ts::routeQueryViewMutation`). Opening the -# server there would give one question two answers, and the client's is the one a person -# experiences. A named door has exactly one answer, and "opening a route widens every field it -# carries unless the cleaner is explicit" is answered by the allow-lists below rather than by hope. - -QUERY_NOTE_MAX = 400 - - -def _artifact_or_404(session, qid, state=None): - """The one wall every artefact door goes through: it exists AND this caller may still see it.""" - state = _state(session) if state is None else state - row = state["views"].get(str(qid)) - if row is None or not _source_still_permitted(session, row.get("source")): - # A caller cannot use an opaque Query key to learn about a revoked artefact. - raise err(404, "unknown_query", "that Query artefact does not exist") - return state, row - - -def _copy_name(name, taken): - """`X copy`, then `X copy 2`, so duplicating twice does not make two rows with one name.""" - base = f"{str(name or 'Query')[:52]} copy" - if base not in taken: - return base[:60] - index = 2 - while f"{base} {index}" in taken and index < 99: - index += 1 - return f"{base} {index}"[:60] - - -@router.patch("/query/{qid}") -def rename_query(qid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Rename a Query view, or give it a description. Nothing else, by construction. - - ⚠ THE TEXT IS THE READER'S, SO IT IS NOT SWEPT FOR DASHES. R6 governs the copy WE write; a note - somebody typed about their own view is theirs, and rewriting its punctuation would be the - product editing a person's words. Model-authored strings are a different case and are swept - where they are produced (`_said`). - """ - body = body if isinstance(body, dict) else {} - state, row = _artifact_or_404(session, qid) - # An ALLOW-LIST, read key by key. A body carrying `view` or `source` changes neither. - patch = {} - if "name" in body: - patch["name"] = " ".join(str(body.get("name") or "").split())[:60] or row.get("name") or "Query" - if "description" in body: - patch["description"] = " ".join(str(body.get("description") or "").split())[:QUERY_NOTE_MAX] - if not patch: - raise err(400, "bad_request", "a Query view takes a new name or a new description") - - def update(raw): - current = _blank_state() - if isinstance(raw, dict): - for key in ("threads", "messages", "citations", "views"): - if isinstance(raw.get(key), dict): - current[key] = copy.deepcopy(raw[key]) - if str(qid) in current["views"]: - current["views"][str(qid)].update(patch) - return current - - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was renamed") - session.runtime.update(_namespace_key(session), update, flush="async") - return _public_view(update(state)["views"][str(qid)]) - - -@router.post("/query/{qid}/duplicate") -def duplicate_query(qid: str, session: Session = Depends(require_session)): - """A second artefact with the same spec and its own identity. - - ⛔ THE CITATIONS ARE COPIED, NOT SHARED. `delete_query` pops an artefact's citations by id, so a - copy pointing at the original's citation ids would lose its provenance the moment the original - was deleted, and a number with no citation is the one thing this module refuses to persist. - """ - state, row = _artifact_or_404(session, qid) - if len(state["views"]) >= MAX_ARTIFACTS: - # R6's second sentence: a cap that cannot be lifted here names its cause and the remedy. - raise err(400, "query_limit", - f"you already have {len(state['views'])} Query views, which is the most one " - f"account can hold. Delete one to make room for this copy.") - new_id = _new_id("query") - copied = copy.deepcopy(row) - copied["id"] = new_id - copied["name"] = _copy_name(row.get("name"), {r.get("name") for r in state["views"].values()}) - copied["createdAt"] = _now_iso() - citations = {} - for citation_id in row.get("citationIds") or []: - source = state["citations"].get(citation_id) - if not source: - continue - fresh_id = _new_id("citation") - citation = copy.deepcopy(source) - citation["id"] = fresh_id - citation["href"] = f"#/query?view={new_id}&citation={fresh_id}" - citations[fresh_id] = citation - copied["citationIds"] = list(citations) - - def update(raw): - current = _blank_state() - if isinstance(raw, dict): - for key in ("threads", "messages", "citations", "views"): - if isinstance(raw.get(key), dict): - current[key] = copy.deepcopy(raw[key]) - current["views"][new_id] = copied - current["citations"].update(citations) - return current - - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was copied") - session.runtime.update(_namespace_key(session), update, flush="async") - return _public_view(copied) - - -# ⛔ A REGISTRY KEY AND A GRID SCOPE ARE DIFFERENT SPELLINGS OF ONE DATABASE, and this product has -# already been bitten by treating them as the same: a `VIEW_OPEN` emit whose topic did not match was -# dropped SILENTLY, 200 OK, and an alert was filed against the wrong database. `customer_data` is -# what the assistant reads; `customer` is what the grid writes. `routes_grid._scope_or_400` refuses -# an unknown scope rather than defaulting it, so this map has to be right rather than nearly right. -_GRID_SCOPE = {"customer_data": "customer", "product_data": "product"} -# The keys a Query spec and a database view actually share. `aggregation` is deliberately absent: -# a database view has no aggregation of its own, which is why the reply below NAMES it as dropped -# instead of letting a "sum of value" answer land as a plain list of rows. -_SAVEABLE = ("visible", "filters", "filterConj", "sorts", "groupBy", "display", "important") - - -def _grid_scope(database): - key = str(database or "").strip() - return _GRID_SCOPE.get(key) or (key if key.startswith("ut_") else "") - - -@router.post("/query/{qid}/save-to-database") -def save_query_to_database(qid: str, session: Session = Depends(require_session)): - """R20: copy this answer into the database's OWN view list, where an ordinary view lives. - - ⛔⛔ `view_upsert` ANSWERS 200 WHILE WRITING NOTHING, IN TWO DIFFERENT WAYS, AND THE WIRE CANNOT - TELL YOU WHICH. `rerender: False` is the documented SUCCESS shape for this event, and it is - also what several silent-refusal branches return: a malformed payload, a missing id or name, an - id that belongs to a view the caller cannot SEE (the wave-9 shared-view guard, which makes - pinned ids effectively tenant-scoped), and a shared view they may see but not edit. So this - door does not believe the response. It READS THE STORE BACK and refuses out loud if the view is - not there. That read is same-process, which is what makes it valid despite `flush="async"`: - the write is visible through the cache long before it is durable. - - ⭐ ONE REQUEST, ONE EVENT. Eighteen POSTs against one JSON document under a coalescing - single-flight once landed ZERO while answering 200 eighteen times; the fix is a batch, never a - retry loop, which treats the symptom and doubles the races. - - ⚠ A FRESH ID, NEVER THE ARTEFACT'S. Reusing the Query id would be a guessable id in a shared - bucket, which is the exact door the wave-9 guard exists to shut. - """ - from routes_grid import grid_events_route - - state, row = _artifact_or_404(session, qid) - scope = _grid_scope((row.get("source") or {}).get("database")) - if not scope: - raise err(409, "unsaveable_source", - "this answer's database does not have a view list to save into.") - spec = row.get("view") or {} - config = {key: copy.deepcopy(spec[key]) for key in _SAVEABLE if spec.get(key) is not None} - config["important"] = spec.get("important") is True - view_id = _new_id("view") - name = " ".join(str(row.get("name") or "Query").split())[:120] or "Query" - - grid_events_route({"scopeKey": scope, "events": [{ - "id": _new_id("event"), "type": "view_upsert", - "view": {"id": view_id, "name": name, "config": config}, - }]}, session) - - document = session.runtime.get(f"{scope}_table_workspace") or {} - landed = (((document.get(session.uname) or {}).get("views") or {}).get(view_id) - if isinstance(document, dict) else None) - if not isinstance(landed, dict): - raise err(409, "view_not_saved", - "the database did not accept this view, so nothing was saved. Open the database " - "and check you can still add a view there.") - - # R6's second sentence, in the payload: what the database's own validator would not take is - # NAMED rather than quietly missing from a view that then looks complete. - stored = landed.get("config") or {} - dropped = [] - if str((spec.get("aggregation") or {}).get("op") or "count") != "count": - dropped.append("the aggregation, which a database view does not carry") - if len(stored.get("filters") or []) != len(config.get("filters") or []): - dropped.append("some of the conditions") - for key in ("visible", "sorts", "groupBy", "display"): - if config.get(key) and not stored.get(key): - dropped.append(key) - return {"scope": scope, "viewId": view_id, "name": landed.get("name") or name, - "dropped": dropped} - - -@router.post("/query/messages/{mid}/rating") -def rate_message(mid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """R20's thumbs, and the ONE thing that stops them being decoration. - - ⛔ A RATING THAT IS WRITTEN AND NEVER READ IS A FLAG SHIPPED WITHOUT ITS WRITER. This one is - read: `_history` turns a thumbs-down into a user turn saying the answer was not helpful, with - the reason if one was given, so the next question in the same thread is answered against that. - The scope is honest and narrow: it steers THIS conversation. It does not train anything, does - not cross threads, and does not cross accounts. - - ⚠ Rating is idempotent and CLEARABLE (`rating: null`), because a mis-click that cannot be taken - back is worse than no control at all, and a flag that can be set and never cleared is the same - defect `important` was fixed for. - """ - body = body if isinstance(body, dict) else {} - mid = str(mid) - raw = body.get("rating") - rating = str(raw or "").strip().lower() - if raw is not None and rating not in QUERY_RATINGS: - raise err(400, "bad_rating", "a rating is up, down, or nothing at all") - state = _state(session) - row = state["messages"].get(mid) - if row is None: - raise err(404, "unknown_message", "that message does not exist") - if row.get("role") != "assistant": - raise err(400, "bad_rating", "only an answer can be rated") - reason = " ".join(str(body.get("reason") or "").split())[:MAX_RATING_REASON] - - def update(raw_state): - current = _blank_state() - if isinstance(raw_state, dict): - for key in ("threads", "messages", "citations", "views"): - if isinstance(raw_state.get(key), dict): - current[key] = copy.deepcopy(raw_state[key]) - message = current["messages"].get(mid) - if message is not None: - message["rating"] = rating if raw is not None else None - # A reason belongs to a thumbs-down; clearing the rating clears it with them. - message["ratingReason"] = reason if (raw is not None and rating == "down") else "" - return current - - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was recorded") - session.runtime.update(_namespace_key(session), update, flush="async") - return _safe(update(state)["messages"][mid]) - - -@router.get("/query/{qid}/export") -def export_query(qid: str, session: Session = Depends(require_session)): - """The artefact's own rows and columns, as CSV. - - ⚠ THE ARTEFACT IS FROZEN; ITS DATA IS NOT, and the difference is worth stating because the two - are easy to conflate. This re-reads the source through the SAME permission wall that built the - artefact and applies the view's own stored predicate, so the file matches what the grid is - showing right now, not the snapshot the stored `numeric` was computed from. That is the useful - answer: a person exports what they are looking at. - """ - state, row = _artifact_or_404(session, qid) - source = row.get("source") or {} - # ⛔ THIS PASSES A STORED OUTPUT BACK INTO AN INPUT SLOT, WHICH IS WORTH STATING RATHER THAN - # HIDING. `source["filters"]` is `deps.assistant_read_scope`'s OWN return value, and that value - # is `validate_assistant_filter(filters, visible)` — the VALIDATED form of whatever the caller - # asked for. MEASURED: `queryApi.submitChat` sends no `filters` key at all, so in production - # this is `None` on every artefact and the round trip never happens. - # ⚠ It is sent anyway, and deliberately, because the alternative FAILS OPEN. If a caller ever - # does narrow a source, dropping the filter here would export MORE rows than the artefact was - # built from, silently. Sending it means a stored filter that cannot be re-validated comes back - # as a NAMED 400 (`assistant_bad_filter`) instead — the fail-closed direction. The gate asserts - # what this door PASSES rather than what the fixture echoes back, because a double built from - # the producer would make this argument valid by construction. - snapshot = assistant_read_scope(session, source.get("database"), fields=None, - filters=source.get("filters") or None) - labels = {field["key"]: str(field.get("label") or field["key"]) for field in snapshot["fields"]} - columns = [key for key in (row["view"].get("visible") or ()) if key in labels] - if not columns: - raise err(409, "nothing_to_export", - "none of this view's columns exist in the database any more, so there is " - "nothing to export. Ask the assistant to build it again.") - buffer = io.StringIO() - writer = csv.writer(buffer, lineterminator="\n") - writer.writerow([labels[key] for key in columns]) - for record in _view_records(snapshot, row["view"]): - writer.writerow(["" if record.get(key) is None else record.get(key) for key in columns]) - stem = re.sub(r"[^A-Za-z0-9]+", "_", str(row.get("name") or "query")).strip("_") or "query" - return Response( - content=buffer.getvalue(), media_type="text/csv; charset=utf-8", - # ⚠ ASCII-only filename: a header carrying a non-ASCII byte is refused by some servers and - # silently mangled by others, and the stem above comes from a model-authored name. - headers={"Content-Disposition": f'attachment; filename="{stem[:60]}.csv"'}) +"""Assistant chat and Query-owned virtual artefacts (W33-T72 / C8). + +Query reads only the detached, permission-filtered snapshot from +``deps.assistant_read_scope``. It owns a separate per-user namespace for threads, messages, +citations and virtual views; it never opens or mutates a source workspace. +""" +import copy +import csv +import datetime as _dt +import hashlib +import io +import json +import os +import re +import sys +import uuid + +from fastapi import APIRouter, Body, Depends, Response + +from deps import (Session, assistant_read_scope, assistant_source_status, err, + require_session) + +router = APIRouter(prefix="/api/v1") + +MAX_QUESTION = 500 +MAX_VISIBLE = 24 +MAX_ARTIFACTS = 200 +# ⭐⭐ R16 — THE THREAD IS REPLAYED, AND THESE TWO NUMBERS ARE THE WHOLE BOUND ON IT. +# The chat was STATELESS: `_call_model` built `[system, user(question)]` and the conversation on +# screen reached the model as nothing at all, so "what about the other one?" could not be answered +# and the assistant could not grill anybody about anything. Replay is bounded twice because the +# two limits fail differently: a turn COUNT stops a long chat costing more every turn, and a +# CHARACTER budget stops one pasted wall of text from being the whole context. Both count from the +# NEWEST end, because that is what a pronoun refers to. +MAX_HISTORY_TURNS = 12 +MAX_HISTORY_CHARS = 6000 +MAX_RATING_REASON = 200 +QUERY_RATINGS = ("up", "down") +MODEL_AUTO = "auto" +# ⭐⭐ WAVE 36 · W36-T35 (R4 / contract C5) — THE LADDER IS DECLARED IN ONE PLACE NOW. +# +# ⛔ THIS BLOCK USED TO BE A SECOND COPY OF THE LADDER, and it said so: *"A SECOND DECLARATION IS A +# DIVERGENCE WAITING TO HAPPEN."* It existed because reading the platform ladder meant +# `import harness.analyst`, MEASURED at **5 to 8 seconds** warm (it pulls `harness.tools`, which +# pulls yaml and the skill recipes) — eight seconds on the first paint of the AI module, to answer +# "which models can this deployment call?". +# +# `providers.LLM_PROVIDERS` is now that answer: a plain declaration in a module this file already +# imports, with no yaml and no skill recipes behind it. So the copy is DELETED rather than kept in +# step — one list, one door (C5), and the eight seconds go with it. +# ⚠ `harness.analyst.PROVIDERS` still exists and is still the ANALYST's ladder. `_ladder_drift()` +# reconciles the two WITHOUT forcing the import, so a divergence is still reported and nobody pays +# the eight seconds to hear about it. +import providers as prov # noqa: E402 + +#: The Messages API version header. ⚠ A DATE, NOT A MODEL: it pins the wire format, not the model, +#: and Anthropic requires it on every request. +ANTHROPIC_VERSION = "2023-06-01" +#: ⛔ THINKING IS ON BY DEFAULT ON THIS MODEL AND `max_tokens` CAPS THINKING PLUS TEXT TOGETHER, so +#: a budget sized around the answer alone truncates mid-thought. It is deliberately NOT disabled: +#: with thinking off, the model occasionally writes a tool call into its visible TEXT instead of +#: emitting a `tool_use` block, and the turn then reports success having called nothing — the exact +#: silent failure this ticket exists to remove. +ANTHROPIC_MAX_TOKENS = int(os.environ.get("AIOS_ANTHROPIC_MAX_TOKENS") or 8000) +#: `low` | `medium` | `high` | `xhigh` | `max`. Medium is the balance for a short, scoped +#: view-building turn; raise it per deployment if the specs come back shallow. +ANTHROPIC_EFFORT = os.environ.get("AIOS_ANTHROPIC_EFFORT") or "medium" + +#: Anything the ANALYST's ladder carries that this one does not, once something has already paid to +#: import it. Empty means "nothing known", never "clean". +_LADDER_DRIFT = [] +QUERY_KINDS = ("grid", "chart", "calendar", "kanban", "timeseries", "map", "list") +QUERY_WORKSPACE_EVENTS = {"view_create", "view_upsert", "view_delete"} +QUERY_EXCLUDED = { + "form": "Forms collect new data and are not a read-only Query artefact.", + "catalog": "Catalog is a source-native presentation that Query cannot safely mutate.", + "swipe": "Swipe is an interactive source-native presentation, not an Assistant output.", +} +_MODE_REFS = { + "kanban": [("stackField", ("select",), True)], + "calendar": [("dateField", ("date",), True)], + "timeseries": [("dateField", ("date",), True)], + "map": [("colorField", ("select",), False), ("sizeField", ("int", "currency", "pct"), False)], +} + + +def _now_iso(): + return _dt.datetime.now(_dt.timezone.utc).isoformat() + + +def _grid(): + import aios_grid + return aios_grid + + +def _safe(value): + """Detach values before they enter a durable Query object or an API reply.""" + return json.loads(json.dumps(value, default=str)) + + +def _namespace_key(session): + """A tenant runtime has one isolated key for each user's Query-owned objects.""" + principal = f"{session.tenant}:{session.uname}".encode("utf-8") + return "query_user_" + hashlib.sha256(principal).hexdigest()[:24] + + +def _blank_state(): + return {"version": 1, "threads": {}, "messages": {}, "citations": {}, "views": {}} + + +def _state(session): + try: + raw = session.runtime.get(_namespace_key(session)) or {} + except Exception: + raw = {} + if not isinstance(raw, dict): + return _blank_state() + out = _blank_state() + for key in out: + if key == "version": + continue + if isinstance(raw.get(key), dict): + out[key] = copy.deepcopy(raw[key]) + return out + + +def _new_id(prefix): + return f"{prefix}_{uuid.uuid4().hex[:16]}" + + +def model_choices(): + """Choices are stable even when one is not configured, so explicit means explicit.""" + return [MODEL_AUTO, *(p.name for p in prov.LLM_PROVIDERS.values())] + + +def _ladder_drift(): + """Does the ANALYST's ladder carry a provider this one does not? Answered only if it is free. + + ⛔ IT MUST NEVER FORCE `import harness.analyst`. That import is 5-8 seconds warm and this + function is reached from `GET /query`, which both AI surfaces call on mount. Reading + `sys.modules` means the reconciliation happens for whoever already paid, and costs nothing for + everybody else — a check that is free is a check that stays on [[limit-with-no-enforcer]]. + """ + analyst = sys.modules.get("harness.analyst") + if analyst is None: + return list(_LADDER_DRIFT) + rows = [row for row in getattr(analyst, "PROVIDERS", []) if isinstance(row, dict)] + known = prov.LLM_PROVIDERS + # ⛔ TWO KINDS OF DRIFT, AND THE SECOND IS THE QUIET ONE. A provider the analyst gained is + # ABSENT here, which a reader can at least notice. An env name that changed on one side leaves + # the provider listed and its availability answered from the wrong variable — present, wrong, + # and silent. Both were checked when this file held its own env map; both are still checked now + # that `providers.LLM_PROVIDERS` holds it, because the two ladders can still disagree. + _LADDER_DRIFT[:] = sorted( + {str(row.get("name")) for row in rows} - set(known) + | {str(row.get("name")) for row in rows + if str(row.get("name")) in known + and str(row.get("env") or "") != known[str(row.get("name"))].env}) + return list(_LADDER_DRIFT) + + +def model_status(): + """Per choice: can this deployment actually CALL it, and if not, why not (C5). + + ⭐⭐ THE SAME DEFECT THE SOURCE CHIPS WERE FIXED FOR IN WAVE 33, ONE CONTROL TO THE LEFT. + `permitted` is not `answerable` for a database; OFFERED is not CALLABLE for a model. The picker + listed every name in the ladder whether or not this deployment holds its key, so choosing one + spent a click and a turn to be told "the selected model is unavailable" by the server, which + knew before the click. The reason travels with the flag, exactly as `sources` does. + + ⭐ W36-T35 ADDS THREE REASONS THE FIRST VERSION COULD NOT EXPRESS, and each is a distinct fact + a picker has to be able to say out loud: + * `outOfCredit` — configured, capable, and SKIPPED because it told us its balance is empty + (R4). Before this, an empty account was indistinguishable from a working one until the + turn came back with `HTTP 402` on the screen. + * `toolCalling` — DECLARED per provider. W36-T34's `done-when` is that a model the ladder + declares incapable of tool calling is not offered for a tool-calling turn; that sentence + needs a declaration to read, and this is it. + * `jsonMode` — the same shape for the other capability staged item 3 named. + + ⚠ IT NAMES NO ENVIRONMENT VARIABLE. "This model is not configured on this deployment" is the + cause a tenant user can act on (ask an admin); the variable name is our deployment's business. + """ + rows_by_name = {row["provider"]: row for row in prov.llm_status()} + live = [name for name, row in rows_by_name.items() + if row["configured"] and row["capable"] and not row["outOfCredit"]] + out = [{"model": MODEL_AUTO, "available": bool(live), + "reason": "" if live else "no model can be reached from this deployment right now", + "toolCalling": True, "jsonMode": True, "outOfCredit": False}] + for name, row in rows_by_name.items(): + if not row["configured"]: + reason = "this model is not configured on this deployment" + elif row["outOfCredit"]: + reason = f"{row['label']} is out of credit, so it is being skipped for now" + elif not row["capable"]: + reason = f"{row['label']} cannot carry this kind of request" + else: + reason = "" + out.append({"model": name, "available": name in live, "reason": reason, + "toolCalling": row["toolCalling"], "jsonMode": row["jsonMode"], + "outOfCredit": row["outOfCredit"]}) + # Anything the ANALYST's ladder carries that Query does not offer, once a chat has taught us. + out.extend({"model": name, "available": False, "toolCalling": False, "jsonMode": False, + "outOfCredit": False, + "reason": "the platform can reach this model but Query does not offer it yet"} + for name in _ladder_drift()) + return out + + +def _providers(model=MODEL_AUTO): + """Auto returns the permitted ladder; an explicit choice returns at most one provider. + + ⛔ THE FILTERING IS `providers.llm_chain`'s, NOT THIS FUNCTION'S — declared-capable AND + configured AND not inside an out-of-credit cooldown, in the ladder's own order (R4). Re-deriving + any of those three here would be a second opinion about which model to call, and the first + place it would disagree is the one nobody watches. + """ + requested = str(model or MODEL_AUTO).strip().lower() + chain = prov.llm_chain("llm_tool_calling") + if requested != MODEL_AUTO: + chain = [p for p in chain if p.name == requested] + return [{"name": p.name, "label": p.label, "env": p.env, "url": p.url, + "model": p.model, "wire": p.wire} for p in chain] + + +def _history(state, thread_id): + """The prior turns of THIS thread, oldest first, in the transport's own message shape. + + ⛔ SORTED BY `(createdAt, role)`, NOT BY `createdAt` ALONE. `_submit` stamps the user turn and + the assistant turn with the SAME `now`, so on timestamp alone the pair ties and their order is + whatever the store's dict iteration happens to give — which is insertion order today and is + not a guarantee anybody wrote down. A replayed conversation with the answers before the + questions is worse than no replay: it reads as coherent and is backwards. + + ⚠ Scoped to one thread by construction. Replaying another thread's turns would leak one chat's + subject into another's answer, which is the kind of wrong that looks like a good answer. + """ + thread_id = str(thread_id or "") + if not thread_id: + return [] + rows = [row for row in state["messages"].values() + if isinstance(row, dict) and str(row.get("threadId") or "") == thread_id] + rows.sort(key=lambda row: (str(row.get("createdAt") or ""), + 0 if row.get("role") == "user" else 1)) + turns = [] + for row in rows[-MAX_HISTORY_TURNS:]: + content = " ".join(str(row.get("content") or "").split()) + if not content: + continue + role = "assistant" if row.get("role") == "assistant" else "user" + turns.append({"role": role, "content": content}) + # ⭐⭐ R20's "self-improving", and it is the ONLY thing that makes a rating more than a + # flag shipped without its writer. A thumbs-down becomes what it actually was: a turn in + # which the reader said the answer was not useful. Modelled as a USER turn rather than by + # editing the assistant's own words, because that is what happened, and because rewriting + # a stored answer to steer the next one is how a transcript stops being a record. + if role == "assistant" and row.get("rating") == "down": + reason = " ".join(str(row.get("ratingReason") or "").split())[:MAX_RATING_REASON] + turns.append({"role": "user", + "content": f"That answer was not helpful. {reason}".strip()}) + kept, spent = [], 0 + for turn in reversed(turns): + spent += len(turn["content"]) + if spent > MAX_HISTORY_CHARS and kept: + break + kept.append(turn) + kept.reverse() + return kept + + +def _spec_schema(field_keys): + col = {"type": "string", "enum": sorted(field_keys)} + return { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": [*QUERY_KINDS, "refused"]}, + "name": {"type": "string"}, + "refusal": {"type": "string"}, + "visible": {"type": "array", "items": col}, + # ⭐⭐ 2026-08-15 (owner: *"if you can build the view, the AI assistant should also be + # able to do it by using our tools on the backend"*). `rhs` and `important` were the + # only two things a person could express through `view_upsert` and this tool could not. + # + # ⛔ WITHOUT `rhs` THE ASSISTANT CANNOT STATE AN ERROR-CATCHER AT ALL — the one shape + # item 8a named by hand (*"price and COGS not matching"*). Every one of the 20 + # `FILTER_OPS` was already reachable, because they all read `value`; comparing a column + # against ANOTHER COLUMN is a different member (`aios_grid._clean_rhs`, CG-9) and it was + # simply absent here, so the model had no way to ask for it and no way to be told why. + # `kind` is `field` ONLY: `measure` needs a window and `stat` needs the population + # vocabulary, neither of which this snapshot-shaped reader carries — offering them + # would be a control that lies, which is the same rule the source chips follow. + "filters": {"type": "array", "items": {"type": "object", "properties": { + "colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)}, + "value": {"type": "string"}, + "rhs": {"type": "object", "properties": { + "kind": {"type": "string", "enum": ["field"]}, "colId": col, + }, "required": ["kind", "colId"]}, + }, "required": ["colId", "op"]}}, + # A personal legibility mark, exactly as `grid_events.view_upsert` treats it (wave 32 + # R5/C4) — not a lock, and no second permission wall. + "important": {"type": "boolean"}, + "filterConj": {"type": "string", "enum": ["and", "or"]}, + "sorts": {"type": "array", "items": {"type": "object", "properties": { + "colId": col, "dir": {"type": "string", "enum": ["asc", "desc"]}, + }, "required": ["colId", "dir"]}}, + "groupBy": col, + "aggregation": {"type": "object", "properties": { + "op": {"type": "string", "enum": ["count", "sum", "avg", "min", "max"]}, + "field": col, + }, "required": ["op"]}, + "stackField": col, + "dateField": col, + "colorField": col, + "sizeField": col, + }, + "required": ["kind"], + } + + +# ══════════ WAVE 36 · W36-T40 — THE GROUNDING CORPUS THIS PRODUCT ALREADY OWNED ════════════════ +# +# ⭐⭐ 25 SENTENCES OF THE OWNER'S OWN SEMANTICS WERE SITTING ON DISK, UNREAD BY THE ASSISTANT. +# `platform/model/metrics/*.yml` carries an `ai_context` on 25 of its 26 registered metrics: what +# "revenue" means here (untaxed, Amazon excluded), that an AR balance takes no date window, that a +# returns figure arriving negative means a lost negation rather than no returns. They were written +# for the AIOS Analyst, which EXIT-6 deleted, and nothing has read them since. +# +# ⛔ THEY ARE VOCABULARY, NOT FIELDS, AND THE PROMPT HAS TO SAY SO. The topics they describe +# (`sales_lines`, `receivables`, `gl_lines`, ...) are NOT databases this assistant can read: its +# sources are the two built-ins plus the `ut_*` tables (`deps.assistant_source_status`). Handing a +# model 25 confident notes about `team_id` and `price_subtotal` without that fence is an invitation +# to filter on a column that is not in the snapshot. +# +# ⛔⛔ AND NO FIGURE INSIDE THEM IS EVER AN ANSWER. Five of the 25 carry a measured landmark +# (`Fisch 2025 = 518`, `$1,447,579.59 vs $1,086,381.35`, `~$192k`, `$3,187,571`, `~2% apart`), +# because a human analyst reading a metric definition wants the order of magnitude. A model that +# repeats one has put an aggregate on a screen that no `validate()` ever tied back to Odoo, which +# is the single thing standing rule 8 forbids. ⚠ THE PROMPT SAYING SO IS NOT THE CONTROL: this +# file already measured a prompt instruction failing on the very next turn (see `_no_dashes`), so +# `verify_query` asks the live model for one of those numbers and asserts it does not come back. +# +# ⚠ ONE READER, NOT TWO. `harness.semantic` is the governed registry, already parsed, validated +# and lru_cached, and `rollup_sql`/`routes_tables` reach it by this same lazy import. Measured +# after `import main`: the import costs **0.003 s** (its whole dependency set, `core.odoo`, `yaml` +# and `modules.sales` included, is already resident) and the first `metrics()` call 0.195 s, once. +# Nothing like the 5 to 8 seconds `harness.analyst` cost, which is why the ladder above stopped +# reading that one and why this does not need a second YAML parser of its own. + + +def _grounding_notes(): + """`[(topic, key, label, sentence), ...]` — this tenant's metric vocabulary, dash free. + + ⛔ FAIL SOFT, AND THE GATE IS WHAT MAKES THAT SAFE. A missing or malformed corpus returns [] + and the assistant answers without the vocabulary, because turning a YAML typo into a dead chat + door is worse than answering with less. That is precisely the silent degradation + `deploy_web.py` warns about in the comment above its own upload of these files, so + `verify_query` derives the expected count FROM the YAML and asserts the built prompt carries + exactly that many: the corpus may go missing, but it may not go missing quietly. + + ⛔ `_no_dashes` ON THE WAY IN, and it is load bearing. 11 of the 25 sentences carry an em dash, + and the prompt they are joining ends with *"Never use an em dash or an en dash"*. Shipping them + raw would demonstrate the forbidden style eleven times in the same breath as the ban. ⚠ AND + `web_prose` STRUCTURALLY CANNOT SEE IT: that gate reads Python string VALUES off the AST, and + these strings live in a data file, so prose loaded at runtime is a hole in rule 6's scanner. + """ + try: + from harness import semantic as sem + registry = sem.metrics() + except Exception: + # Named metrics are a data-layer concern; a chat turn is not the place to surface a + # corpus problem to a person who asked about their own table. + return [] + notes = [] + for key, metric in (registry or {}).items(): + if not isinstance(metric, dict): + continue + sentence = _no_dashes(str(metric.get("ai_context") or "")).strip() + if sentence: + # ⛔ THE LABEL GETS THE SAME TREATMENT, and it is not belt-and-braces: measured on + # this corpus, `sales_orders.orders_invoiced` is labelled "Orders — fully invoiced" + # and TWO em dashes reached the built prompt while every sentence was clean. One + # field of a row scrubbed and its neighbour not is [[one-question-two-normalizers]] + # at the smallest possible scale, so the rule applies to the whole row. + notes.append((str(metric.get("topic") or ""), str(key), + _no_dashes(str(metric.get("label") or key)).strip(), sentence)) + return notes + + +def _grounding_block(): + """The system-prompt section, or "" when this deployment has no corpus to state.""" + notes = _grounding_notes() + if not notes: + return "" + lines = "\n".join(f"- {topic}.{key} ({label}): {sentence}" + for topic, key, label, sentence in notes) + return f""" +BUSINESS VOCABULARY ({len(notes)} notes this tenant wrote about their own reporting): +{lines} + +Read those for what a WORD means here: which rows a number counts, which reading of an ambiguous +question is the house one, what a figure deliberately leaves out. They describe the finance topics, +not the database above, so never treat a name in that list as a field and never filter on anything +absent from FIELDS. + +Some notes quote a MEASUREMENT taken in an earlier year: a total, a customer count, a gap between +two figures. Never repeat one of those to the reader, not even labelled as old or approximate. A +number on screen is read as the answer whatever sits beside it, and nothing checked that one today. +When this database cannot produce the figure somebody asked for, say only that. Codes and formats +in the notes are different and you may use them freely: a team_id, a percentage scale, which field +carries what. Every number you state comes from build_view over the snapshot above. +""" + + +def _system_prompt(snapshot): + fields = snapshot["fields"] + cols = json.dumps([{"key": field["key"], "label": field.get("label", field["key"]), + "type": field.get("type", "text")} for field in fields], separators=(",", ":")) + return f"""You are the assistant inside this product, talking with one person about exactly one +database they already have permission to read. Reply in plain prose, and keep it short. + +DATABASE: {snapshot['database']} +SNAPSHOT VERSION: {json.dumps(snapshot['source_version'], default=str)} +PERMISSION-FILTERED RECORD COUNT: {len(snapshot['records'])} +FIELDS: {cols} +VIEW KINDS: {", ".join(QUERY_KINDS)} + +You may answer in words alone, and you may ask ONE short question back when the request is +ambiguous: which field is meant, which period, whether they want every record or a narrower set. +Prefer asking over guessing whenever the answer would change what you build. Earlier turns of this +conversation are above; use them, and read a pronoun as referring to what was just discussed. + +Call build_view ONLY when the person wants to SEE records: a table, a chart, a board, a list. Do +not call it to explain something, to confirm something, or to ask your question. When you do call +it you may also write one sentence saying what you built. + +Inside build_view: choose visible fields, supported filters and an optional aggregation. Use count +for record counts; sum, avg, min and max require one numeric field. The server computes and cites +every number from this exact snapshot. Never write SQL, invent fields, or name another database. +Return kind=refused with one plain sentence if the database cannot answer. + +To compare one column against ANOTHER column rather than a typed value, give the filter an rhs of +{{"kind":"field","colId":""}} and omit value. That is how you express questions like +"priced below what it costs us". Both columns must be in the FIELDS list above. +Set important=true when the view is one somebody should be chased about: an error, a mismatch, or +money at risk. Leave it out otherwise. +{_grounding_block()} +Write with ordinary punctuation. Never use an em dash or an en dash: use a comma, a colon or a full +stop instead.""" + + +_FAILED_GEN = re.compile(r"(\{.*?\})\s*", re.S) + + +def _spec_from_400(body): + try: + value = ((json.loads(body) or {}).get("error") or {}).get("failed_generation") or "" + raw = (_FAILED_GEN.search(str(value)) or [str(value).strip()])[1] + result = json.loads(raw) + return result if isinstance(result, dict) else None + except Exception: + return None + + +# ⚠ BUILT FROM `chr`, AND A UNICODE ESCAPE IS NOT ENOUGH. `web_prose` reads a Python +# string's VALUE off the AST, not its spelling in the source, so an escape and the character +# itself are the SAME finding to it. Composing the class at runtime means no string literal in +# this file holds a dash, which is true rather than merely quiet: the only dashes in this module +# are the ones being removed. +_DASH = "[" + chr(0x2014) + chr(0x2013) + "]" + + +def _no_dashes(text): + """R6 applied where it is the ONLY place it can be applied: the model's own words. + + ⛔⛔ THE PROMPT INSTRUCTION IS NOT ENFORCEMENT, AND THIS IS MEASURED, NOT ANTICIPATED. The + system prompt ends with *"Never use an em dash or an en dash"* and the very next live turn came + back with *"Which view type would you like—grid, chart, list, or another?"* (cerebras, + 2026-08-16). Model prose reaches the screen exactly as a string literal does, and `web_prose` + scans SOURCE, so it cannot see a dash that arrives at runtime: the sweep every lane is doing + this wave is undone by our own assistant unless it is undone here. + + ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE. "10–20" means "10 to 20"; rewriting it as + "10, 20" states two numbers where the model stated a span, which is a wrong answer rather than + a punctuation fix. It gets its own rule, first. + """ + text = str(text or "") + text = re.sub(rf"(?<=\d)\s*{_DASH}\s*(?=\d)", " to ", text) + text = re.sub(rf"\s*{_DASH}\s*(?=[,.;:!?])", "", text) # abutting punctuation: it just goes + text = re.sub(rf"(?<=[,;:])\s*{_DASH}\s*", " ", text) # already punctuated: one space + return re.sub(rf"\s*{_DASH}\s*", ", ", text) + + +def _said(raw): + """The model's own words: one line of whitespace, no dash, and empty means nothing was said.""" + return _no_dashes(" ".join(str(raw or "").split())).strip() or None + + +def _from_chat(answer, provider): + """An injected transport may answer with a SPEC (a dict) or with PROSE (a string). + + ⛔ THE BRANCH BELONGS HERE, NOT AT THE CALLER. `_validate`'s first line refuses anything that + is not a dict, with "the assistant did not answer with a view" — so a prose answer passed + straight through would arrive as a REFUSAL, styled as one, and every conversation check would + pass against the wrong path while looking green [[one-question-two-normalizers]]. + """ + if isinstance(answer, str): + return None, _said(answer), provider, None + return answer, None, provider, None + + +# ══════════════════ WAVE 36 · W36-T35 — TWO WIRES, ONE LOOP (ruling R4) ═════════════════════════ +# +# ⛔ THE SECOND WIRE IS NOT A SECOND CODE PATH BY CHOICE — Anthropic's Messages API genuinely is a +# different request: `x-api-key` instead of a bearer, the system prompt as a TOP-LEVEL field rather +# than a message, `input_schema` instead of `function.parameters`, and typed content BLOCKS instead +# of `choices[0].message`. What is kept single is everything around it: one ladder, one loop, one +# meter, one set of sentences. The wire is a pair of pure functions the loop dispatches on. +# +# ⚠ WHY RAW HTTP AND NOT THE `anthropic` SDK, which is the documented default. The SDK is not in +# `aios-web/api/requirements.txt`, so it is NOT in the deployed image — adding it is a dependency +# on a Space this lane cannot rebuild or test, mid-wave. `requests` is already there and is already +# how the three sibling providers in this same loop are called. Booked for the integrator as a +# `PENDING:`; the day it lands, `_anthropic_request`/`_anthropic_read` are the two functions that +# go, and nothing else moves. + + +def _openai_request(provider, messages, tools): + """The OpenAI-compatible `/chat/completions` shape — cerebras, groq, openrouter.""" + return { + "url": provider["url"], + "headers": {"Authorization": f"Bearer {os.environ[provider['env']]}"}, + "json": {"model": provider["model"], "messages": messages, "tools": tools, + # ⛔⛔ `auto`, NOT `required`, AND THAT ONE WORD IS R16. Under `required` the model + # had to emit a `build_view` call for EVERY turn: it could not answer a question, + # could not ask one back, and could not decline to build. "The AI chat must TALK + # with the user, not only build queries, and decide for itself whether to create a + # query view" is unreachable with the tool forced, whatever the prompt says. + "tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200}, + } + + +def _anthropic_request(provider, messages, tools): + """The Messages API shape — built by `providers.anthropic_request`, never re-derived here. + + ⛔ ONE WIRE, TWO DOORS. `ai_review.draft_flow` calls the same builder; the owner quoted an + error from each of these paths in one sentence, and two implementations of one API is the + defect this product keeps finding under a different name. + ⚠ THIS DOOR SENDS `effort` AND THE DRAFTER DOES NOT — `output_config.effort` errors on Haiku + 4.5, which is the tier the drafter runs on. Model-gated, so it is the caller's call. + """ + return prov.anthropic_request( + model=provider["model"], key=os.environ[provider["env"]], + system=None, messages=messages, tools=tools, + max_tokens=ANTHROPIC_MAX_TOKENS, tool_choice="auto", effort=ANTHROPIC_EFFORT) + + +def _openai_read(body): + """`(spec, said, refusal)` out of an OpenAI-compatible answer.""" + answer = ((body.get("choices") or [{}])[0] or {}).get("message") or {} + said = _said(answer.get("content")) + calls = answer.get("tool_calls") or [] + if calls: + try: + spec = json.loads(calls[0]["function"].get("arguments") or "{}") + except (TypeError, ValueError, KeyError): + return None, said, None + return (spec if isinstance(spec, dict) else None), said, None + return None, said, None + + +def _anthropic_read(body): + """`(spec, said, refusal)` — the shared reader, with this door's own dash normaliser applied.""" + text, spec, refused = prov.anthropic_read(body) + if refused: + return None, None, refused + return spec, _said(text), None + + +_WIRES = {"openai": (_openai_request, _openai_read), + "anthropic": (_anthropic_request, _anthropic_read)} + + +def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None): + """Return ``(spec, said, provider, reason)`` without any source-data fallback. + + ``said`` is what the assistant SAID: the whole answer when it did not build a view, and the + sentence beside the view when it built one and talked as well. ``reason`` is set only when + something went wrong, so a prose ANSWER and a transport FAILURE are distinguishable one layer + up rather than both arriving as a bare sentence. + + ⭐⭐ W35-T31 · CONTRACT C7 (R9) — ``session`` IS HERE ONLY SO THE METER CAN BE TOLD WHOSE CALL + THIS WAS, and PRD amendment A1 is why it is an argument rather than ambient state: a + ``ContextVar`` bound in ``deps.require_session`` reads back ``None`` inside this function on the + SAME thread, so a ledger built on one would have recorded nothing and shown a dashboard of + zeros indistinguishable from a quiet week. + + ⚠ OPTIONAL, DELIBERATELY. ``verify_query`` calls this function directly with an injected + ``chat`` and no session, and a required argument would have made every one of those calls a + signature change. An omitted session is COUNTED in ``usage_ledger.UNATTRIBUTED`` and reported + to an admin, never dropped. + """ + requested = str(model or MODEL_AUTO).strip().lower() + if requested not in model_choices(): + return None, f"the selected model ({requested or model}) is unavailable", None, "model_unavailable" + tools = [{"type": "function", "function": { + "name": "build_view", "description": "Emit a virtual-view spec or a refusal.", + "parameters": _spec_schema([field["key"] for field in snapshot["fields"]]), + }}] + messages = [{"role": "system", "content": _system_prompt(snapshot)}, + *(history or []), + {"role": "user", "content": question}] + if chat is not None: + provider = requested if requested != MODEL_AUTO else "injected" + return _from_chat(chat(messages, tools), provider) + + providers = _providers(requested) + if not providers: + # ⛔ THREE DIFFERENT EMPTINESSES, THREE DIFFERENT SENTENCES (R4). "Not configured" and + # "every account is out of credit" are the same empty list and completely different + # problems: one needs an administrator, the other needs a card. Answering both with one + # sentence is how the owner ended up reading `HTTP 402` to find out which. + broke = [row for row in prov.llm_status() if row["outOfCredit"]] + if requested != MODEL_AUTO: + row = next((r for r in prov.llm_status() if r["provider"] == requested), None) + if row and row["outOfCredit"]: + sentence = f"{row['label']} is out of credit, so it cannot answer right now" + else: + sentence = f"the selected model ({requested}) is unavailable" + return None, sentence, None, "model_unavailable" + if broke: + names = ", ".join(row["label"] for row in broke) + return (None, f"every model this deployment can reach is out of credit ({names}). " + f"Top one up and it will answer again", None, "no_credit") + return None, "the assistant is not configured on this deployment", None, "not_configured" + + import requests + + import usage_ledger + + def _book(provider, body): + """⭐⭐ C7 — ONE LEDGER LINE PER PROVIDER RESPONSE THIS FUNCTION READS. + + ⛔ HERE, INSIDE THE LADDER, AND NOT AT THE RETURN. The ladder tries providers in order and + a failed one has already SPENT tokens at that vendor; booking only the winner would report + a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads + gets a line, including the one whose answer turned out to be empty. + ⚠ `tokens_from` reads `input_tokens`/`output_tokens` as well as the OpenAI spellings, so + the Messages API body is measured by the SAME reader with no branch here. That is not luck + — it is why that function takes a key LIST. + ⚠ `record` never raises, by its own contract, so this cannot break the assistant. + """ + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record( + "assistant", provider["name"], provider["model"], ins, outs, + total=usage_ledger.total_from(body), + st=getattr(session, "runtime", None), user=getattr(session, "uname", "")) + + # ⛔ EVERY FAILURE IS COLLECTED, NOT JUST THE LAST ONE. The owner's own complaint was a LIST + # ("cerebras: HTTP 402; groq: HTTP 404; openrouter: HTTP 402") and the list is the useful part + # — it is what tells an operator whether one vendor is down or the account is empty everywhere. + # What changes is that each entry is now a sentence rather than a status code. + trouble = [] + for provider in providers: + build, read = _WIRES.get(provider["wire"], _WIRES["openai"]) + try: + request = build(provider, messages, tools) + response = requests.post(request["url"], timeout=90, + headers=request["headers"], json=request["json"]) + except Exception as exc: # noqa: BLE001 + trouble.append(f"{provider['label']} could not be reached ({type(exc).__name__})") + continue + + if response.status_code == 400 and provider["wire"] == "openai": + # `_FAILED_GEN`'s recovery, kept for the wire that needs it and NOT extended to the one + # that does not: Anthropic answers a tool call as a typed block or not at all, so a 400 + # there is a malformed REQUEST and recovering a spec out of it would be reading tea + # leaves. + spec = _spec_from_400(response.text) + if spec is not None: + return spec, None, provider["name"], None + + if response.status_code != 200: + # ⭐⭐ R4's SECOND CLAUSE, AND THE ONLY PLACE IT CAN BE LEARNED: a provider only tells + # you the balance is empty by refusing. The memo is written here so the NEXT turn skips + # it instead of paying the round trip again. + if prov.is_credit_failure(response.status_code, response.text): + prov.mark_no_credit(provider["name"]) + trouble.append(prov.refusal_sentence(provider["name"], response.status_code, + response.text)) + continue + + try: + body = response.json() + # C7: booked from the RAW body, before anything below can raise on its shape. A + # provider that answered 200 and billed for it has spent tokens whether or not this + # function can read what it said. + _book(provider, body) + spec, said, refused = read(body) + if refused: + trouble.append(f"{provider['label']}: {refused}") + continue + if spec is not None: + # The prose rides ALONG with the view when the model wrote both, so the reader gets + # a sentence instead of `_explain`'s machine description of its own output. + return spec, said, provider["name"], None + if said: + return None, said, provider["name"], None + # ⚠ Neither a call nor a word is a FAILED turn, not a silent one: falling through to + # the next provider is right, and swallowing it as an empty answer would show the + # reader a blank reply [[empty-answer-vs-unfinished-answer]]. + trouble.append(f"{provider['label']} answered with nothing") + except Exception as exc: # noqa: BLE001 + trouble.append(f"{provider['label']} sent something unreadable " + f"({type(exc).__name__})") + if requested != MODEL_AUTO: + return (None, f"the selected model ({requested}) could not answer: " + + (trouble[0] if trouble else "it did not respond"), None, "model_unavailable") + return (None, "the assistant could not be reached just now: " + + ("; ".join(trouble) if trouble else "no provider answered"), + None, "provider_unreachable") + + +def _validate(spec, fields): + """Return a cleaned, source-independent virtual-view config or a named refusal.""" + if not isinstance(spec, dict): + return None, "the assistant did not answer with a view", "no_spec" + by_key = {str(field.get("key")): str(field.get("type") or "text") for field in fields} + keys = set(by_key) + kind = spec.get("kind") + if kind == "refused": + # Model-authored, so it faces the same R6 wall the model's chat prose does. + return None, _said(spec.get("refusal")) or "this database cannot answer that question", "model_refused" + if kind in QUERY_EXCLUDED or kind not in QUERY_KINDS: + return None, "that kind of view cannot be built from this question", "unsupported_kind" + + named = set(spec.get("visible") or ()) + for item in spec.get("filters") or (): + if isinstance(item, dict) and item.get("colId"): + named.add(str(item["colId"])) + # ⛔ THE RIGHT-HAND COLUMN IS A COLUMN AND MUST FACE THE SAME `missing` CHECK. Collecting + # only the left side is what makes D-229 possible one layer down: `clean_filter_tree` does + # NOT drop a leaf whose field-rhs names a column that does not exist — it keeps the leaf, + # strips the `rhs`, blanks the value, and `filter_sql.is_rule_active` then reports the rule + # INACTIVE. An inactive rule narrows nothing, so "margin under 10%" would come back as a + # view listing the ENTIRE catalogue under an error-catcher's name, with nothing red. + # Naming it here turns that into the ordinary "this database does not have: X" refusal. + rhs = item.get("rhs") if isinstance(item, dict) else None + if isinstance(rhs, dict) and rhs.get("colId"): + named.add(str(rhs["colId"])) + for item in spec.get("sorts") or (): + if isinstance(item, dict) and item.get("colId"): + named.add(str(item["colId"])) + for key in ("groupBy", "stackField", "dateField", "colorField", "sizeField"): + if spec.get(key): + named.add(str(spec[key])) + raw_aggregation = spec.get("aggregation") or {"op": "count"} + if isinstance(raw_aggregation, dict) and raw_aggregation.get("field"): + named.add(str(raw_aggregation["field"])) + missing = sorted(named - keys) + if missing: + return None, "this database does not have: " + ", ".join(missing), "unknown_columns" + + visible = [str(key) for key in (spec.get("visible") or ()) if str(key) in keys][:MAX_VISIBLE] + if not visible: + return None, "that question did not name any fields to show", "no_columns" + raw_filters = [item for item in (spec.get("filters") or ()) if isinstance(item, dict)] + filters = _grid().clean_filter_tree(raw_filters, keys) + if len(filters) != len(raw_filters): + return None, "part of that filter is unsupported", "filter_dropped" + # ⛔⛔ A SECOND, NARROWER CHECK, AND THE LENGTH CHECK ABOVE CANNOT DO ITS JOB (D-229). + # A dropped leaf changes the COUNT; a stripped `rhs` does not — the leaf survives, so + # `len(filters) == len(raw_filters)` and the refusal above never fires. The failure is + # therefore silent in exactly the direction that matters: the condition stops narrowing and + # the view answers with every record. Assert the member survived, per leaf. + # ⚠ The `named` pass above already refuses an rhs naming a column this database lacks, so + # reaching here means something ELSE stripped it (a type the comparand cannot take, a future + # `_clean_rhs` rule). Both doors, because the two catch different causes and the cost of + # missing this one is a wrong answer that looks right. + for sent, kept in zip(raw_filters, filters): + if sent.get("rhs") and not kept.get("rhs"): + return None, "that column cannot be compared against another column", "rhs_dropped" + + aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {} + op = str(aggregation.get("op") or "").lower() + field = aggregation.get("field") + if op not in {"count", "sum", "avg", "min", "max"}: + return None, "the assistant gave an unsupported aggregation", "bad_aggregation" + if op == "count": + field = None + elif field not in keys or by_key.get(field) not in {"int", "currency", "pct"}: + return None, "that aggregation needs one visible numeric field", "bad_aggregation" + + display = {"mode": kind} + for ref, families, required in _MODE_REFS.get(kind, ()): + value = spec.get(ref) + if required and not value: + return None, f"a {kind} view needs {ref}", "missing_ref" + if value and by_key.get(value) not in families: + return None, f"{ref} has the wrong field type", "wrong_ref_type" + if value: + display[ref] = value + cleaned_display = _grid()._clean_display(display, keys) if kind != "grid" else None + if kind != "grid" and not cleaned_display: + return None, f"this product could not build a {kind} view", "display_dropped" + + return { + "kind": kind, + # The view NAME is model-authored too, and it is the string that ends up in the rail, in + # the flyout and on the artefact card. R6 reaches it here or nowhere. + "name": (_said(spec.get("name")) or "Query")[:60], + "visible": visible, + "filters": filters, + "filterConj": "or" if spec.get("filterConj") == "or" else "and", + "sorts": [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} + for item in (spec.get("sorts") or []) if isinstance(item, dict) + and item.get("colId") in keys][:3], + "groupBy": spec.get("groupBy") if spec.get("groupBy") in keys else None, + "aggregation": {"op": op, "field": field}, + "display": cleaned_display, + # ⚠ `is True`, not truthy, and UNCONDITIONAL — the same two rules `grid_events.view_upsert` + # follows for this key. `is True` so a model emitting the string "false" does not mark a + # view; unconditional so the mark is REMOVABLE rather than a flag that can be set and never + # cleared (a key written only when present leaves a stored `true` alive forever). + "important": spec.get("important") is True, + }, None, None + + +def _explain(view, fields): + label = {field["key"]: str(field.get("label") or field["key"]) for field in fields} + visible = ", ".join(label.get(key, key) for key in view["visible"][:6]) + result = f"{view['kind']} view of {visible}" + if view["filters"]: + result += "; filtered records only" + if view.get("groupBy"): + result += f"; grouped by {label.get(view['groupBy'], view['groupBy'])}" + agg = view["aggregation"] + if agg["op"] != "count": + result += f"; {agg['op']} of {label.get(agg['field'], agg['field'])}" + return result + "." + + +def _said_fallback(artifact): + """What a build turn SAYS when the provider volunteered no sentence of its own. + + ⚠ MEASURED, 2026-08-16, cerebras: a view-shaped question comes back as a tool call with + `content: null`. So this is the sentence a person reads on MOST build turns, not a rare + fallback, and R16 ("the AI chat must TALK with the user") is decided here rather than in the + prompt. `_explain` used to be it, and it is a receipt for our own output: *"grid view of + Company, Owner, Deal value; filtered records only."* It keeps its real job ON THE ARTEFACT, + where it labels the thing it describes. + """ + numeric = artifact.get("numeric") or {} + value = numeric.get("value") + if value is None: + return f"I built {artifact['name']}." + if isinstance(value, float) and value.is_integer(): + value = int(value) + number = f"{value:,}" if isinstance(value, (int, float)) else str(value) + label = " ".join(str(numeric.get("label") or "Records").split()) + return f"I built {artifact['name']}. {label}: {number}." + + +def _numeric_result(view, records): + aggregation = view["aggregation"] + if aggregation["op"] == "count": + return {"label": "Matching records", "value": len(records), "contributing_record_count": len(records)} + values = [] + for record in records: + try: + value = record.get(aggregation["field"]) + if value is not None and not isinstance(value, bool): + values.append(float(value)) + except (TypeError, ValueError): + continue + if not values: + return {"label": aggregation["op"], "value": None, "contributing_record_count": 0} + op = aggregation["op"] + value = {"sum": sum(values), "avg": sum(values) / len(values), "min": min(values), "max": max(values)}[op] + return {"label": f"{op.title()} of {aggregation['field']}", "value": value, + "contributing_record_count": len(values)} + + +def _referenced_fields(view): + """The citation names every source field that affected the displayed result.""" + out = list(view.get("visible") or ()) + for node in view.get("filters") or (): + if isinstance(node, dict) and node.get("colId"): + out.append(str(node["colId"])) + for node in view.get("sorts") or (): + if isinstance(node, dict) and node.get("colId"): + out.append(str(node["colId"])) + for key in ("groupBy",): + if view.get(key): + out.append(str(view[key])) + aggregation = view.get("aggregation") or {} + if aggregation.get("field"): + out.append(str(aggregation["field"])) + return list(dict.fromkeys(out)) + + +def _effective_filters(snapshot, view): + """Keep the source request and generated-view predicates distinct in provenance.""" + return { + "source": _safe(snapshot.get("filters")), + "view": {"conj": view.get("filterConj", "and"), + "nodes": _safe(view.get("filters") or [])}, + } + + +def _view_records(snapshot, view): + """Apply the exact validated virtual-view filter before calculating a cited number.""" + nodes = view.get("filters") or [] + if not nodes: + return list(snapshot["records"]) + from harness import filter_eval + tree = {"conj": view.get("filterConj", "and"), "nodes": nodes} + return [row for row in snapshot["records"] + if filter_eval.matches(tree, row, snapshot["fields"])] + + +def _citation(citation_id, snapshot, view, numeric, view_id): + return { + "id": citation_id, + "href": f"#/query?view={view_id}&citation={citation_id}", + "database": snapshot["database"], + "snapshot": {"kind": snapshot["source_kind"], "version": _safe(snapshot["source_version"])}, + "fields": [field["key"] for field in snapshot["fields"] + if field.get("key") in set(_referenced_fields(view))], + "filters": _effective_filters(snapshot, view), + "permission_scope_applied": snapshot["permission_scope_applied"], + "aggregation": _safe(view["aggregation"]), + "contributing_record_count": numeric["contributing_record_count"], + "retrieved_at": snapshot["retrieved_at"], + } + + +def _citation_complete(citation): + """A numeric result is not publishable without complete provenance.""" + required = {"database", "snapshot", "fields", "filters", "aggregation", + "contributing_record_count", "retrieved_at", "href"} + snapshot = citation.get("snapshot") if isinstance(citation, dict) else None + return (isinstance(citation, dict) and required <= set(citation) + and bool(citation["database"]) and isinstance(snapshot, dict) + and "version" in snapshot and bool(citation["retrieved_at"]) + and isinstance(citation["fields"], list) + and isinstance(citation["aggregation"], dict)) + + +def _public_view(view): + source = view["source"] + # ⭐⭐ W35-T25 · CONTRACT C4 — `edited` IS DERIVED, NEVER STORED, and that is deliberate. + # A stored boolean beside two specs is a third source of truth that can disagree with both; + # the only honest answer to "has this been changed" is "compare it". It also means a Revert + # that restores the spec clears the badge by construction rather than by remembering to. + original = view.get("original_spec") + return { + "id": view["id"], "viewId": view["id"], "scope": source["database"], + "name": view["name"], "description": str(view.get("description") or ""), + "kind": view["view"]["kind"], "question": view["question"], + "explain": view["explain"], "threadId": view["threadId"], "createdAt": view["createdAt"], + "virtual": True, "source": _safe(source), "view": _safe(view["view"]), + "citationIds": list(view["citationIds"]), "numeric": _safe(view["numeric"]), + # ⚠ AN ARTEFACT MADE BEFORE THIS WAVE HAS NO ORIGINAL, so it reports `edited: false` and + # sends no `original_spec` — and the client shows neither the badge nor Revert. C4 is + # explicit that this is the right answer: a Revert with nothing to revert to is worse + # than an absent one, and it is the case an NC in `verify_query` covers. + "original_spec": _safe(original) if isinstance(original, dict) else None, + "edited": bool(isinstance(original, dict) and _safe(view["view"]) != _safe(original)), + } + + +def _source_still_permitted(session, source, permitted=None): + """A persisted artefact never outlives the caller's current data permission. + + ⚠ `permitted` is the BATCH answer from `deps.assistant_source_status` — one rows-free + resolution for every source at once, rather than one whole-document read per saved view. That + function's docstring carries the measurement. The single-source path below stays for callers + holding exactly one artefact (the workspace-event door), and asks the same helper. + + ⛔ `permitted`, NEVER `answerable`. A source whose rows are served through the connector mirror + cannot be ASKED and can still be SEEN: the artefact was built from a snapshot that was legal + when it was taken, and hiding it because the reader can no longer make a NEW one would read as + deletion. The two verdicts are separate for that reason. + """ + database = (source or {}).get("database") if isinstance(source, dict) else None + if not database: + return False + if permitted is None: + permitted = {key for key, row in assistant_source_status(session, [database]).items() + if row["permitted"]} + return database in permitted + + +def _public_state(state, session): + # ONE rows-free batch answers both questions: which sources this caller may still see (which + # artefacts stay listed) and which of them can actually be asked (which chips are live). + status = assistant_source_status(session) + for row in state["views"].values(): + key = (row.get("source") or {}).get("database") if isinstance(row.get("source"), dict) else None + if key and key not in status: + # An artefact whose source is no longer enumerable still gets a verdict rather than a + # KeyError — it resolves to "not permitted" and the artefact drops out, which is the + # same answer the per-view wall gave. + status.update(assistant_source_status(session, [key])) + permitted = {key for key, row in status.items() if row["permitted"]} + allowed_views = [row for row in state["views"].values() + if _source_still_permitted(session, row.get("source"), permitted)] + allowed_ids = {row["id"] for row in allowed_views} + allowed_citations = {citation_id for row in allowed_views + for citation_id in row.get("citationIds") or []} + threads = sorted(state["threads"].values(), key=lambda row: row.get("updatedAt", ""), reverse=True) + messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", "")) + views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True) + messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids] + return {"threads": _safe(threads), "messages": _safe(messages), "views": views, + "citations": _safe([row for key, row in state["citations"].items() + if key in allowed_citations]), "models": model_choices(), + # The model picker's own data, the same shape `sources` uses for databases. + "modelStatus": model_status(), + # The chip row's own data: a source this caller holds but cannot ask, and WHY. + # ⭐⭐ D-276 — `or row.get("visible")`. A source the caller can open elsewhere in the + # product but the assistant cannot read is now LISTED with its cause, instead of + # vanishing from a picker that shows every other database they hold. It arrives with + # `answerable: False` and a `reason`, which is the SAME shape the mirror-served grids + # already use, so the chip row needs no new state to render it. + # ⛔ `permitted` STILL GATES ARTEFACTS — `allowed_views` above is unchanged. This widens + # what is DESCRIBED, never what can be read or kept. + "sources": _safe([{"database": key, "answerable": row["answerable"], + "reason": row["reason"]} + for key, row in sorted(status.items()) + if row["permitted"] or row.get("visible")])} + + +@router.get("/query") +def list_queries(session: Session = Depends(require_session)): + return _public_state(_state(session), session) + + +# ⭐⭐ W35-T25 · CONTRACT C4 (owner item 4 / R3) — WHICH SPEC MEMBERS AN EDIT MAY MOVE. +# +# ⛔ AN ALLOW-LIST, AND IT IS THE SECURITY BOUNDARY OF THIS WHOLE TICKET. The client sends a +# `SavedView`, a shape it composes itself, and merging it wholesale would let a caller rewrite +# `kind` (which decides the renderer and the citation's own claim), `aggregation` (the number this +# artefact CITED) or `name`. R3 opens the view SPEC: *"filters, sort, group, visible columns, +# widths"*, plus the row height and column order that carry them. Nothing else. +# +# ⚠ `aggregation` IS DELIBERATELY ABSENT even though a person can change it on an ordinary grid. +# The artefact's citation records `contributing_record_count` and an op computed from THIS +# aggregation; letting an edit move it would leave a cited number describing a calculation the +# artefact no longer performs, which is the one thing every provenance rule in this module exists +# to prevent. +# +# ⚠ A COMMENT, NOT A BARE MODULE-LEVEL STRING. `verify_prose` reads a free-floating string literal +# as candidate copy, so writing this as a `"""..."""` above the constant put an em dash into the +# gate's own census as a 296th finding. A `#` block is out of scope by construction. +QUERY_EDITABLE_SPEC = ("visible", "order", "widths", "filters", "filterConj", "sorts", + "groupBy", "rowHeightMode", "frozenCount", "colorBy", "display") + + +def _clean_spec_edit(config, spec, fields): + """Merge a client view config onto the artefact's spec, cleaned against ITS OWN fields. + + ⛔ THE FIELD KEYS COME FROM THE STORED SOURCE, never from the request. The artefact carries + the snapshot's field list, so this needs no database read and cannot be widened by a caller + naming a column the snapshot did not have. + """ + keys = {str(field.get("key")) for field in (fields or []) if isinstance(field, dict)} + out = copy.deepcopy(spec) if isinstance(spec, dict) else {} + if not isinstance(config, dict): + return out + for member in QUERY_EDITABLE_SPEC: + if member not in config: + continue + value = config[member] + if member in ("visible", "order"): + cleaned = [str(key) for key in value if str(key) in keys] if isinstance(value, list) else [] + # ⚠ An EMPTY visible list is refused rather than stored: `_validate` already treats + # "no columns" as a refusal at creation, and a grid showing nothing is not an edit + # somebody meant to make. + if member == "visible" and not cleaned: + continue + out[member] = cleaned[:MAX_VISIBLE] + elif member == "widths": + out[member] = {str(key): int(width) for key, width in value.items() + if str(key) in keys and isinstance(width, (int, float)) + and 0 < float(width) <= 2000} if isinstance(value, dict) else {} + elif member == "filters": + out[member] = _grid().clean_filter_tree( + [item for item in value if isinstance(item, dict)], keys) if isinstance(value, list) else [] + elif member == "filterConj": + out[member] = "or" if value == "or" else "and" + elif member == "sorts": + out[member] = [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} + for item in value if isinstance(item, dict) and item.get("colId") in keys][:3] \ + if isinstance(value, list) else [] + elif member in ("groupBy", "colorBy"): + out[member] = value if value in keys else None + elif member == "rowHeightMode": + out[member] = value if value in ("short", "medium", "tall", "extra") else None + elif member == "frozenCount": + out[member] = max(0, min(6, int(value))) if isinstance(value, (int, float)) else 0 + elif member == "display": + out[member] = _grid()._clean_display(value, keys) if isinstance(value, dict) else out.get("display") + return out + + +@router.post("/query/{qid}/events") +def mutate_query_workspace(qid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """The only grid-mutation transport for a virtual Query workspace. + + ⭐⭐ W35-T25 (owner item 4 / R2) — `view_upsert` ON THE ARTEFACT ITSELF IS NOW ACCEPTED. It + used to 409 `query_workspace_immutable` for everything but a delete, on the reading that an AI + artefact is a snapshot. R2 replaces that reading: the SPEC is the reader's to shape, the + CITATION and the numbers behind it are not. `QUERY_EDITABLE_SPEC` above is where that line is + drawn, and `original_spec` is what makes the change undoable. + + Creating a SECOND view inside the workspace is still refused — an artefact holds exactly one — + and deleting the caller's personal artefact still removes it. The binding key, not a + client-supplied source scope, identifies it. + """ + qid = str(qid) + body = body if isinstance(body, dict) else {} + binding = body.get("workspaceBinding") + event = body.get("event") + if not (isinstance(binding, dict) and binding.get("kind") == "query" + and str(binding.get("key") or "") == qid): + raise err(400, "query_workspace_mismatch", + "the Query workspace binding must name this virtual artefact") + if not isinstance(event, dict) or str(event.get("type") or "") not in QUERY_WORKSPACE_EVENTS: + raise err(400, "unsupported_query_workspace_event", + "Query accepts only view_create, view_upsert, or view_delete events") + + state = _state(session) + view = state["views"].get(qid) + if view is None or not _source_still_permitted(session, view.get("source")): + # A caller cannot use an opaque Query key to learn about a revoked artefact. + raise err(404, "unknown_query", "that Query artefact does not exist") + + event_type = str(event["type"]) + if event_type == "view_upsert": + sent = event.get("view") + sent = sent if isinstance(sent, dict) else {} + # ⛔ THE ID IS CHECKED THE SAME WAY THE DELETE'S IS. An upsert naming a different view is + # a create wearing an update's name, and a Query workspace holds exactly one view. + if str(sent.get("id") or "") != qid: + raise err(400, "query_workspace_mismatch", + "a Query view edit must name the same virtual artefact as its binding") + spec = _clean_spec_edit(sent.get("config"), view.get("view"), (view.get("source") or {}).get("fields")) + + def apply_edit(raw): + current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() + row = (current.get("views") or {}).get(qid) + if not isinstance(row, dict): + return current + # ⚠ BACKFILLED HERE, and only when absent: an artefact created before this wave has no + # original, and the FIRST edit is the last moment its pre-edit spec still exists. Not + # backfilling would leave it permanently unrevertable; backfilling unconditionally + # would make Revert restore the latest edit. + if not isinstance(row.get("original_spec"), dict): + row["original_spec"] = _safe(row.get("view")) + row["view"] = _safe(spec) + current["views"][qid] = row + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was saved") + session.runtime.update(_namespace_key(session), apply_edit, flush="async") + return {"workspaceBinding": {"kind": "query", "key": qid}, "event": event_type, + "view": _public_view(apply_edit(_state(session))["views"][qid])} + + if event_type != "view_delete": + raise err(409, "query_workspace_immutable", + "a Query workspace holds one view, so it cannot take another") + if str(event.get("viewId") or "") != qid: + raise err(400, "query_workspace_mismatch", + "a Query delete must name the same virtual artefact as its binding") + + deleted = delete_query(qid, session) + return {"workspaceBinding": {"kind": "query", "key": qid}, + "event": event_type, **deleted} + + +@router.post("/query/{qid}/revert") +def revert_query(qid: str, session: Session = Depends(require_session)): + """⭐⭐ CONTRACT C4 (R3) — restore the AI's original SPEC, and only the spec. + + ⛔ WHAT THIS DOES NOT DO, which the client's confirm says out loud BEFORE it acts: it does not + undo anything the reader changed in the SOURCE database. A Query view is live now, so a cell + edit made through it is a real write to a real record — reverting a view's filters cannot and + must not walk those back. R3 is explicit that the dialog states this before it acts, because a + Revert that silently leaves data changed is worse than one that never offered. + + ⚠ 409, not 404, when there is no original: the artefact exists and is readable, and the caller + asked for something that does not exist FOR IT. A 404 would say the artefact is gone. + """ + qid = str(qid) + # ⚠ `_artifact_or_404` returns `(state, row)`, not the row. Read it as a pair. + _state_now, view = _artifact_or_404(session, qid) + original = view.get("original_spec") + if not isinstance(original, dict): + raise err(409, "no_original_spec", + "this view was created before the assistant kept an original, so there is " + "nothing to revert to") + + def restore(raw): + current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() + row = (current.get("views") or {}).get(qid) + if not isinstance(row, dict): + return current + row["view"] = _safe(row.get("original_spec")) + current["views"][qid] = row + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was reverted") + session.runtime.update(_namespace_key(session), restore, flush="async") + return _public_view(restore(_state(session))["views"][qid]) + + +def _sources(body, target, session): + raw = body.get("sources") if isinstance(body, dict) else None + sources = [str(item).strip() for item in raw] if isinstance(raw, list) else [] + sources = list(dict.fromkeys(item for item in sources if item)) + if target not in sources: + sources.insert(0, target) + # Source chips are a permissioned selection, not untrusted labels remembered in a thread. + # The model still receives ONLY `target`'s snapshot below; these calls do not pass a second + # source to it and D's helper refuses cache misses, mirrors and unshared databases. + for source in sources: + assistant_read_scope(session, source, fields=[], filters=None) + return sources + + +def _submit(body, session, chat=None): + body = body if isinstance(body, dict) else {} + question = " ".join(str(body.get("question") or "").split()) + target = str(body.get("database") or body.get("scope") or "").strip() + selected_model = str(body.get("model") or MODEL_AUTO).strip().lower() + if not question: + raise err(400, "bad_request", "no question was asked") + if len(question) > MAX_QUESTION: + raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters") + if not target: + raise err(400, "bad_request", "one target database must be selected") + if selected_model not in model_choices(): + raise err(400, "unknown_model", "that model is not available in Query") + + # This call happens before model selection and is the only permitted source-data read. + snapshot = assistant_read_scope(session, target, fields=None, filters=body.get("filters")) + sources = _sources(body, target, session) + state = _state(session) + thread_id = str(body.get("threadId") or "").strip() + if thread_id and thread_id not in state["threads"]: + raise err(404, "unknown_thread", "that chat does not exist") + if not thread_id: + thread_id = _new_id("thread") + now = _now_iso() + user_message_id = _new_id("message") + assistant_message_id = _new_id("message") + # R16: the turns already on screen go with the question. `state` was read above, BEFORE this + # turn's own messages exist, so the replay is strictly the prior conversation. + # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s + # docstring for why it is an argument and not ambient state (PRD amendment A1). + spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat, + history=_history(state, thread_id), session=session) + view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason) + + artifact = None + citation = None + if view is not None: + if len(state["views"]) >= MAX_ARTIFACTS: + raise err(400, "query_limit", f"you already have {len(state['views'])} Query artefacts") + view_id = _new_id("query") + citation_id = _new_id("citation") + numeric = _numeric_result(view, _view_records(snapshot, view)) + citation = _citation(citation_id, snapshot, view, numeric, view_id) + if not _citation_complete(citation): + raise RuntimeError("Query refused to persist an incomplete numeric citation") + source = {"database": snapshot["database"], "label": snapshot["label"], + "source_kind": snapshot["source_kind"], "source_version": _safe(snapshot["source_version"]), + "retrieved_at": snapshot["retrieved_at"], + "fields": _safe(snapshot["fields"]), "filters": _safe(snapshot["filters"]), + "permission_scope_applied": snapshot["permission_scope_applied"]} + artifact = {"id": view_id, "threadId": thread_id, "name": view["name"], "description": "", + "question": question, + "explain": _explain(view, snapshot["fields"]), "createdAt": now, "source": source, + "view": _safe(view), "citationIds": [citation_id], "numeric": numeric, + # ⭐⭐ W35-T25 · CONTRACT C4 (R3) — THE AI'S OWN SPEC, WRITTEN ONCE, HERE. + # R2 makes a Query view editable, so `view` moves from now on. This is the copy + # "Revert to AI original" restores, and the thing `edited` is measured against. + # ⛔ WRITTEN AT CREATE AND NOWHERE ELSE. Re-stamping it on any later write would + # make Revert restore the most recent edit — a Revert that reverts to nothing, + # which C4 names as worse than no Revert at all. + # ⚠ `_safe(view)` twice, not the same object twice: `view` is mutable and a + # shared reference would let an edit rewrite the original through the alias. + "original_spec": _safe(view), + "model": provider, "requestedModel": selected_model} + + assistant_message = { + "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now, + "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"), + "targetDatabase": target, "requestedModel": selected_model, "model": provider, + "reason": refusal_code, "viewId": artifact["id"] if artifact else None, + "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None, + } + user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now, + "content": question, "sources": sources, "targetDatabase": target, + "requestedModel": selected_model} + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + thread = current["threads"].get(thread_id) or {"id": thread_id, "createdAt": now} + # ⚠ THE FIRST QUESTION NAMES THE CHAT, and R16 is what makes that matter. While every turn + # was a fresh one-shot the title could only be the last thing asked; now that a thread is a + # conversation, retitling it on every follow-up renames the history entry out from under + # the reader, and "and the other one?" is a useless name for anything. + thread.update({"updatedAt": now, "title": thread.get("title") or question[:80], "sources": sources, + "model": selected_model, "activeViewId": artifact["id"] if artifact else thread.get("activeViewId")}) + current["threads"][thread_id] = thread + current["messages"][user_message_id] = user_message + current["messages"][assistant_message_id] = assistant_message + if artifact: + current["views"][artifact["id"]] = artifact + current["citations"][citation["id"]] = citation + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; no chat was saved") + session.runtime.update(_namespace_key(session), update, flush="async") + return {"thread": _safe(update(state)["threads"][thread_id]), "userMessage": _safe(user_message), + "message": _safe(assistant_message), + "view": _public_view(artifact) if artifact else None, "citations": [citation] if citation else []} + + +@router.post("/query/chat") +def submit_chat(body: dict = Body(default=None), session: Session = Depends(require_session)): + return _submit(body, session) + + +@router.delete("/query/threads/{tid}") +def delete_thread(tid: str, session: Session = Depends(require_session)): + """Remove one of the caller's own chats, and only the chat. + + ⛔ THE ARTEFACTS SURVIVE, AND THAT IS THE POINT. A Query view is a durable object in its own + right — it appears in Query's rail, other people's links can point at it, and the owner's + instruction was that AI views LIVE THERE rather than inside the conversation that happened to + produce them. Cascading the delete would make tidying up a chat silently destroy work. Views + are deleted from Query's own rail, one at a time, by `delete_query` below. + + ⚠ It exists because the history had no prune. `delete_query` removed a view and left its + thread, so the panel could only ever grow — a navigation you cannot manage is half a + navigation, and a person clearing test chats found that out first. + """ + tid = str(tid) + state = _state(session) + if tid not in state["threads"]: + raise err(404, "unknown_thread", "that chat does not exist") + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + current["threads"].pop(tid, None) + for message_id in [key for key, row in current["messages"].items() + if (row or {}).get("threadId") == tid]: + current["messages"].pop(message_id, None) + # ⚠ THE VIEWS' `threadId` IS LEFT EXACTLY AS IT WAS, dangling. Blanking it looks tidier and + # is a data-loss bug: `queryApi.saved()` refuses a row with an empty `threadId`, so every + # artefact built in the deleted chat would silently vanish from Query's rail — the exact + # "tidying up destroys work" outcome this door is written not to have + # [[a-record-can-outlive-its-subject]]. Nothing resolves the id but the chat panel, which + # only ever looks up the thread it is showing. + return current + + session.runtime.update(_namespace_key(session), update, flush="async") + return {"deleted": tid} + + +@router.delete("/query/{qid}") +def delete_query(qid: str, session: Session = Depends(require_session)): + qid = str(qid) + state = _state(session) + if qid not in state["views"]: + raise err(404, "unknown_query", "that Query artefact does not exist") + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + view = current["views"].pop(qid, None) + for citation_id in (view or {}).get("citationIds") or []: + current["citations"].pop(citation_id, None) + return current + + session.runtime.update(_namespace_key(session), update, flush="async") + return {"deleted": qid} + + +# ══ R20 — A QUERY VIEW HAS EVERY FUNCTION A DATABASE VIEW HAS ═══════════════════════════════════ +# +# ⛔⛔ THE DECISION THIS TICKET ASKED FOR, WRITTEN DOWN RATHER THAN IMPLIED, because it is the line +# every later reader will need and there is no other place it exists. +# +# **IMMUTABLE, and unchanged by R20:** the generated SPEC and its provenance. `view` (kind, visible, +# filters, sorts, aggregation, display, important), `source` (the snapshot, its version, its +# retrieval time, the permission scope), `question`, `citationIds`, `numeric`. A cited number is +# only worth citing if the thing it was computed from cannot be edited underneath it, so +# `mutate_query_workspace` still answers 409 `query_workspace_immutable` for `view_create` and +# `view_upsert`, and `QUERY_MUTATION_POLICY` on the client is untouched. +# +# **OPENED:** `name` and `description`. They are LABELS, not the answer: changing them cannot make +# a citation wrong. Plus a server-side DUPLICATE (the client never supplies a spec, so a copy is a +# copy) and an EXPORT, which is a read. +# +# ⛔ AND THE REASON THESE ARE NAMED DOORS RATHER THAN AN OPENED `view_upsert`, which is what the +# ticket's `how:` first suggested: the client refuses `view_upsert` on a Query binding LOCALLY, +# before any request leaves the browser (`queryPreview.ts::routeQueryViewMutation`). Opening the +# server there would give one question two answers, and the client's is the one a person +# experiences. A named door has exactly one answer, and "opening a route widens every field it +# carries unless the cleaner is explicit" is answered by the allow-lists below rather than by hope. + +QUERY_NOTE_MAX = 400 + + +def _artifact_or_404(session, qid, state=None): + """The one wall every artefact door goes through: it exists AND this caller may still see it.""" + state = _state(session) if state is None else state + row = state["views"].get(str(qid)) + if row is None or not _source_still_permitted(session, row.get("source")): + # A caller cannot use an opaque Query key to learn about a revoked artefact. + raise err(404, "unknown_query", "that Query artefact does not exist") + return state, row + + +def _copy_name(name, taken): + """`X copy`, then `X copy 2`, so duplicating twice does not make two rows with one name.""" + base = f"{str(name or 'Query')[:52]} copy" + if base not in taken: + return base[:60] + index = 2 + while f"{base} {index}" in taken and index < 99: + index += 1 + return f"{base} {index}"[:60] + + +@router.patch("/query/{qid}") +def rename_query(qid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Rename a Query view, or give it a description. Nothing else, by construction. + + ⚠ THE TEXT IS THE READER'S, SO IT IS NOT SWEPT FOR DASHES. R6 governs the copy WE write; a note + somebody typed about their own view is theirs, and rewriting its punctuation would be the + product editing a person's words. Model-authored strings are a different case and are swept + where they are produced (`_said`). + """ + body = body if isinstance(body, dict) else {} + state, row = _artifact_or_404(session, qid) + # An ALLOW-LIST, read key by key. A body carrying `view` or `source` changes neither. + patch = {} + if "name" in body: + patch["name"] = " ".join(str(body.get("name") or "").split())[:60] or row.get("name") or "Query" + if "description" in body: + patch["description"] = " ".join(str(body.get("description") or "").split())[:QUERY_NOTE_MAX] + if not patch: + raise err(400, "bad_request", "a Query view takes a new name or a new description") + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + if str(qid) in current["views"]: + current["views"][str(qid)].update(patch) + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was renamed") + session.runtime.update(_namespace_key(session), update, flush="async") + return _public_view(update(state)["views"][str(qid)]) + + +@router.post("/query/{qid}/duplicate") +def duplicate_query(qid: str, session: Session = Depends(require_session)): + """A second artefact with the same spec and its own identity. + + ⛔ THE CITATIONS ARE COPIED, NOT SHARED. `delete_query` pops an artefact's citations by id, so a + copy pointing at the original's citation ids would lose its provenance the moment the original + was deleted, and a number with no citation is the one thing this module refuses to persist. + """ + state, row = _artifact_or_404(session, qid) + if len(state["views"]) >= MAX_ARTIFACTS: + # R6's second sentence: a cap that cannot be lifted here names its cause and the remedy. + raise err(400, "query_limit", + f"you already have {len(state['views'])} Query views, which is the most one " + f"account can hold. Delete one to make room for this copy.") + new_id = _new_id("query") + copied = copy.deepcopy(row) + copied["id"] = new_id + copied["name"] = _copy_name(row.get("name"), {r.get("name") for r in state["views"].values()}) + copied["createdAt"] = _now_iso() + citations = {} + for citation_id in row.get("citationIds") or []: + source = state["citations"].get(citation_id) + if not source: + continue + fresh_id = _new_id("citation") + citation = copy.deepcopy(source) + citation["id"] = fresh_id + citation["href"] = f"#/query?view={new_id}&citation={fresh_id}" + citations[fresh_id] = citation + copied["citationIds"] = list(citations) + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + current["views"][new_id] = copied + current["citations"].update(citations) + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was copied") + session.runtime.update(_namespace_key(session), update, flush="async") + return _public_view(copied) + + +# ⛔ A REGISTRY KEY AND A GRID SCOPE ARE DIFFERENT SPELLINGS OF ONE DATABASE, and this product has +# already been bitten by treating them as the same: a `VIEW_OPEN` emit whose topic did not match was +# dropped SILENTLY, 200 OK, and an alert was filed against the wrong database. `customer_data` is +# what the assistant reads; `customer` is what the grid writes. `routes_grid._scope_or_400` refuses +# an unknown scope rather than defaulting it, so this map has to be right rather than nearly right. +_GRID_SCOPE = {"customer_data": "customer", "product_data": "product"} +# The keys a Query spec and a database view actually share. `aggregation` is deliberately absent: +# a database view has no aggregation of its own, which is why the reply below NAMES it as dropped +# instead of letting a "sum of value" answer land as a plain list of rows. +_SAVEABLE = ("visible", "filters", "filterConj", "sorts", "groupBy", "display", "important") + + +def _grid_scope(database): + key = str(database or "").strip() + return _GRID_SCOPE.get(key) or (key if key.startswith("ut_") else "") + + +@router.post("/query/{qid}/save-to-database") +def save_query_to_database(qid: str, session: Session = Depends(require_session)): + """R20: copy this answer into the database's OWN view list, where an ordinary view lives. + + ⛔⛔ `view_upsert` ANSWERS 200 WHILE WRITING NOTHING, IN TWO DIFFERENT WAYS, AND THE WIRE CANNOT + TELL YOU WHICH. `rerender: False` is the documented SUCCESS shape for this event, and it is + also what several silent-refusal branches return: a malformed payload, a missing id or name, an + id that belongs to a view the caller cannot SEE (the wave-9 shared-view guard, which makes + pinned ids effectively tenant-scoped), and a shared view they may see but not edit. So this + door does not believe the response. It READS THE STORE BACK and refuses out loud if the view is + not there. That read is same-process, which is what makes it valid despite `flush="async"`: + the write is visible through the cache long before it is durable. + + ⭐ ONE REQUEST, ONE EVENT. Eighteen POSTs against one JSON document under a coalescing + single-flight once landed ZERO while answering 200 eighteen times; the fix is a batch, never a + retry loop, which treats the symptom and doubles the races. + + ⚠ A FRESH ID, NEVER THE ARTEFACT'S. Reusing the Query id would be a guessable id in a shared + bucket, which is the exact door the wave-9 guard exists to shut. + """ + from routes_grid import grid_events_route + + state, row = _artifact_or_404(session, qid) + scope = _grid_scope((row.get("source") or {}).get("database")) + if not scope: + raise err(409, "unsaveable_source", + "this answer's database does not have a view list to save into.") + spec = row.get("view") or {} + config = {key: copy.deepcopy(spec[key]) for key in _SAVEABLE if spec.get(key) is not None} + config["important"] = spec.get("important") is True + view_id = _new_id("view") + name = " ".join(str(row.get("name") or "Query").split())[:120] or "Query" + + grid_events_route({"scopeKey": scope, "events": [{ + "id": _new_id("event"), "type": "view_upsert", + "view": {"id": view_id, "name": name, "config": config}, + }]}, session) + + document = session.runtime.get(f"{scope}_table_workspace") or {} + landed = (((document.get(session.uname) or {}).get("views") or {}).get(view_id) + if isinstance(document, dict) else None) + if not isinstance(landed, dict): + raise err(409, "view_not_saved", + "the database did not accept this view, so nothing was saved. Open the database " + "and check you can still add a view there.") + + # R6's second sentence, in the payload: what the database's own validator would not take is + # NAMED rather than quietly missing from a view that then looks complete. + stored = landed.get("config") or {} + dropped = [] + if str((spec.get("aggregation") or {}).get("op") or "count") != "count": + dropped.append("the aggregation, which a database view does not carry") + if len(stored.get("filters") or []) != len(config.get("filters") or []): + dropped.append("some of the conditions") + for key in ("visible", "sorts", "groupBy", "display"): + if config.get(key) and not stored.get(key): + dropped.append(key) + return {"scope": scope, "viewId": view_id, "name": landed.get("name") or name, + "dropped": dropped} + + +@router.post("/query/messages/{mid}/rating") +def rate_message(mid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """R20's thumbs, and the ONE thing that stops them being decoration. + + ⛔ A RATING THAT IS WRITTEN AND NEVER READ IS A FLAG SHIPPED WITHOUT ITS WRITER. This one is + read: `_history` turns a thumbs-down into a user turn saying the answer was not helpful, with + the reason if one was given, so the next question in the same thread is answered against that. + The scope is honest and narrow: it steers THIS conversation. It does not train anything, does + not cross threads, and does not cross accounts. + + ⚠ Rating is idempotent and CLEARABLE (`rating: null`), because a mis-click that cannot be taken + back is worse than no control at all, and a flag that can be set and never cleared is the same + defect `important` was fixed for. + """ + body = body if isinstance(body, dict) else {} + mid = str(mid) + raw = body.get("rating") + rating = str(raw or "").strip().lower() + if raw is not None and rating not in QUERY_RATINGS: + raise err(400, "bad_rating", "a rating is up, down, or nothing at all") + state = _state(session) + row = state["messages"].get(mid) + if row is None: + raise err(404, "unknown_message", "that message does not exist") + if row.get("role") != "assistant": + raise err(400, "bad_rating", "only an answer can be rated") + reason = " ".join(str(body.get("reason") or "").split())[:MAX_RATING_REASON] + + def update(raw_state): + current = _blank_state() + if isinstance(raw_state, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw_state.get(key), dict): + current[key] = copy.deepcopy(raw_state[key]) + message = current["messages"].get(mid) + if message is not None: + message["rating"] = rating if raw is not None else None + # A reason belongs to a thumbs-down; clearing the rating clears it with them. + message["ratingReason"] = reason if (raw is not None and rating == "down") else "" + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was recorded") + session.runtime.update(_namespace_key(session), update, flush="async") + return _safe(update(state)["messages"][mid]) + + +@router.get("/query/{qid}/export") +def export_query(qid: str, session: Session = Depends(require_session)): + """The artefact's own rows and columns, as CSV. + + ⚠ THE ARTEFACT IS FROZEN; ITS DATA IS NOT, and the difference is worth stating because the two + are easy to conflate. This re-reads the source through the SAME permission wall that built the + artefact and applies the view's own stored predicate, so the file matches what the grid is + showing right now, not the snapshot the stored `numeric` was computed from. That is the useful + answer: a person exports what they are looking at. + """ + state, row = _artifact_or_404(session, qid) + source = row.get("source") or {} + # ⛔ THIS PASSES A STORED OUTPUT BACK INTO AN INPUT SLOT, WHICH IS WORTH STATING RATHER THAN + # HIDING. `source["filters"]` is `deps.assistant_read_scope`'s OWN return value, and that value + # is `validate_assistant_filter(filters, visible)` — the VALIDATED form of whatever the caller + # asked for. MEASURED: `queryApi.submitChat` sends no `filters` key at all, so in production + # this is `None` on every artefact and the round trip never happens. + # ⚠ It is sent anyway, and deliberately, because the alternative FAILS OPEN. If a caller ever + # does narrow a source, dropping the filter here would export MORE rows than the artefact was + # built from, silently. Sending it means a stored filter that cannot be re-validated comes back + # as a NAMED 400 (`assistant_bad_filter`) instead — the fail-closed direction. The gate asserts + # what this door PASSES rather than what the fixture echoes back, because a double built from + # the producer would make this argument valid by construction. + snapshot = assistant_read_scope(session, source.get("database"), fields=None, + filters=source.get("filters") or None) + labels = {field["key"]: str(field.get("label") or field["key"]) for field in snapshot["fields"]} + columns = [key for key in (row["view"].get("visible") or ()) if key in labels] + if not columns: + raise err(409, "nothing_to_export", + "none of this view's columns exist in the database any more, so there is " + "nothing to export. Ask the assistant to build it again.") + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + writer.writerow([labels[key] for key in columns]) + for record in _view_records(snapshot, row["view"]): + writer.writerow(["" if record.get(key) is None else record.get(key) for key in columns]) + stem = re.sub(r"[^A-Za-z0-9]+", "_", str(row.get("name") or "query")).strip("_") or "query" + return Response( + content=buffer.getvalue(), media_type="text/csv; charset=utf-8", + # ⚠ ASCII-only filename: a header carrying a non-ASCII byte is refused by some servers and + # silently mangled by others, and the stem above comes from a model-authored name. + headers={"Content-Disposition": f'attachment; filename="{stem[:60]}.csv"'}) diff --git a/api/routes_records.py b/api/routes_records.py index c37f74b261d63bb390db45660363e8d94ff796d4..ef0bb2c8153db1e40b2656a715f545d80b1fe22b 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -1,211 +1,211 @@ -"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. - -⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a -customer-shaped wall bolted to the module import line: `_in_book` asked -`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and -typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the -panel answered "that customer is not in your book" — the owner's report. The dangerous half is -the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment -is filed against somebody's customer where the whole team can read it. - -THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the -events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall -per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving -one here: - - customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant - product `routes_products.scoped_pool` behind the `product_data` grant - ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall - (404 unknown / 403 not yours — a user table has no module grant). - ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every - ROW to derive a pid set this module discards, and RAISES 409 on a - read-through grid past one window, which is why the record drawer painted - an error page on `ut_odoo_gl_lines`. - -⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a -section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning -another session's gate mid-wave — the WALL is the query parameter, not the word. - -⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every -shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a -400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for -the same reason: a typo served as `customer` answers a question nobody asked). -""" -from fastapi import APIRouter, Body, Depends, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table -#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. -_CUSTOMER_SCOPES = ("", "customer", "cohort") - - -def _scope_or_400(raw): - scope = str(raw or "customer").strip().lower() - if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): - return "customer" if scope in _CUSTOMER_SCOPES else scope - raise err(400, "bad_scope", - "scope must be customer, cohort, product or a ut_ database — refusing to guess") - - -def _pool_or_refuse(session: Session, scope: str): - """The pids this session may attach comments to ON THIS DATABASE — grant wall included. - - Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, - so a caller who may not open the surface never learns anything about the row they asked about. - - ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past - one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns - `frozenset()` for both — see the branch below. - ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that - do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` - that REPLACES this function with its own lambda. A gate's test double is a caller - ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a - bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no - failing name. Change the shape here and that double changes with it. - """ - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - session.require(PRODUCT_MODULE) - pids, _team, _rows, _fields = scoped_pool(session) - # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes - # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised - # a `TypeError` — a 500 on every product comment — while both other branches worked. A - # return shape is a contract even when the function is private. - return (pids, False) - if scope.startswith("ut_"): - # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS - # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the - # rows routes do. - # - # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. - # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on - # a read-through grid larger than one window it RAISES `409 window_required`. So opening the - # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel - # that renders comments about ONE row it already has. `scoped_pids` answers the identical - # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes - # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` - # used to become openable on those two grids. - from routes_tables import scoped_pids - - limits = [] - pids, _fields, _defn = scoped_pids(session, scope, limits=limits) - # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is - # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a - # database with no rows and for a read-through grid too big to enumerate — it distinguishes - # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move - # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which - # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). - # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database - # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the - # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside - # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. - return (pids, bool(limits)) - from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids - - session.require(CUSTOMER_MODULE) - return (frozenset(allowed_pids(session)), False) - - -def _in_book(pid, session, scope): - pids, unbounded = _pool_or_refuse(session, scope) - if unbounded: - # The table wall passed and the row set is larger than this process will enumerate. Said - # out loud rather than silently admitting: R6's second sentence is that a limit which - # cannot be removed gets REPORTED, and this is the one place the report has no envelope to - # ride in. - print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " - f"admitting on the table wall alone, per D-72") - return - if pid not in pids: - # 403, not 404: the record may exist, but this session may not inspect it. - raise err(403, "out_of_scope", "that record is not in your book") - - -def _unavailable(): - return err( - 503, - "store_unavailable", - "record comments are temporarily unavailable — no change was saved", - ) - - -# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD -# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the -# wall was always the query param). The old path stays as an ALIAS because the shipped client -# still calls it; verify_api pins the canonical path AND that the alias answers, so removing -# the alias later is a decision, never an accident. -@router.get("/records/{pid}/comments") -@router.get("/customers/{pid}/comments") -def comments(pid: int, scope: str = Query(default="customer"), - session: Session = Depends(require_session)): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - rows = record_comments.list_comments(session.runtime, pid, scope=scope) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comments": rows} - - -@router.post("/records/{pid}/comments", status_code=201) -@router.post("/customers/{pid}/comments", status_code=201) -def create_comment( - pid: int, - body: dict = Body(default=None), - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - comment = record_comments.add_comment( - session.runtime, - pid, - (body or {}).get("body"), - session.uname, - session.user.get("name") or session.uname, - scope=scope, - ) - except ValueError as exc: - raise err(400, "bad_comment", str(exc)) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comment": comment} - - -@router.delete("/records/{pid}/comments/{comment_id}") -@router.delete("/customers/{pid}/comments/{comment_id}") -def remove_comment( - pid: int, - comment_id: str, - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - deleted = record_comments.delete_comment( - session.runtime, - pid, - comment_id, - session.uname, - admin=session.admin, - scope=scope, - ) - except record_comments.CommentForbidden: - raise err(403, "comment_forbidden", "only the author may delete this comment") - except record_comments.CommentsUnavailable: - raise _unavailable() - if not deleted: - raise err(404, "comment_not_found", "that comment no longer exists") - return {"ok": True, "id": comment_id} +"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. + +⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a +customer-shaped wall bolted to the module import line: `_in_book` asked +`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and +typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the +panel answered "that customer is not in your book" — the owner's report. The dangerous half is +the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment +is filed against somebody's customer where the whole team can read it. + +THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the +events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall +per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving +one here: + + customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant + product `routes_products.scoped_pool` behind the `product_data` grant + ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall + (404 unknown / 403 not yours — a user table has no module grant). + ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every + ROW to derive a pid set this module discards, and RAISES 409 on a + read-through grid past one window, which is why the record drawer painted + an error page on `ut_odoo_gl_lines`. + +⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a +section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning +another session's gate mid-wave — the WALL is the query parameter, not the word. + +⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every +shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a +400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for +the same reason: a typo served as `customer` answers a question nobody asked). +""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table +#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. +_CUSTOMER_SCOPES = ("", "customer", "cohort") + + +def _scope_or_400(raw): + scope = str(raw or "customer").strip().lower() + if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): + return "customer" if scope in _CUSTOMER_SCOPES else scope + raise err(400, "bad_scope", + "scope must be customer, cohort, product or a ut_ database — refusing to guess") + + +def _pool_or_refuse(session: Session, scope: str): + """The pids this session may attach comments to ON THIS DATABASE — grant wall included. + + Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, + so a caller who may not open the surface never learns anything about the row they asked about. + + ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past + one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns + `frozenset()` for both — see the branch below. + ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that + do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` + that REPLACES this function with its own lambda. A gate's test double is a caller + ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a + bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no + failing name. Change the shape here and that double changes with it. + """ + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + session.require(PRODUCT_MODULE) + pids, _team, _rows, _fields = scoped_pool(session) + # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes + # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised + # a `TypeError` — a 500 on every product comment — while both other branches worked. A + # return shape is a contract even when the function is private. + return (pids, False) + if scope.startswith("ut_"): + # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS + # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the + # rows routes do. + # + # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. + # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on + # a read-through grid larger than one window it RAISES `409 window_required`. So opening the + # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel + # that renders comments about ONE row it already has. `scoped_pids` answers the identical + # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes + # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` + # used to become openable on those two grids. + from routes_tables import scoped_pids + + limits = [] + pids, _fields, _defn = scoped_pids(session, scope, limits=limits) + # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is + # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a + # database with no rows and for a read-through grid too big to enumerate — it distinguishes + # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move + # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which + # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). + # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database + # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the + # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside + # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. + return (pids, bool(limits)) + from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids + + session.require(CUSTOMER_MODULE) + return (frozenset(allowed_pids(session)), False) + + +def _in_book(pid, session, scope): + pids, unbounded = _pool_or_refuse(session, scope) + if unbounded: + # The table wall passed and the row set is larger than this process will enumerate. Said + # out loud rather than silently admitting: R6's second sentence is that a limit which + # cannot be removed gets REPORTED, and this is the one place the report has no envelope to + # ride in. + print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " + f"admitting on the table wall alone, per D-72") + return + if pid not in pids: + # 403, not 404: the record may exist, but this session may not inspect it. + raise err(403, "out_of_scope", "that record is not in your book") + + +def _unavailable(): + return err( + 503, + "store_unavailable", + "record comments are temporarily unavailable — no change was saved", + ) + + +# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD +# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the +# wall was always the query param). The old path stays as an ALIAS because the shipped client +# still calls it; verify_api pins the canonical path AND that the alias answers, so removing +# the alias later is a decision, never an accident. +@router.get("/records/{pid}/comments") +@router.get("/customers/{pid}/comments") +def comments(pid: int, scope: str = Query(default="customer"), + session: Session = Depends(require_session)): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + rows = record_comments.list_comments(session.runtime, pid, scope=scope) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comments": rows} + + +@router.post("/records/{pid}/comments", status_code=201) +@router.post("/customers/{pid}/comments", status_code=201) +def create_comment( + pid: int, + body: dict = Body(default=None), + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + comment = record_comments.add_comment( + session.runtime, + pid, + (body or {}).get("body"), + session.uname, + session.user.get("name") or session.uname, + scope=scope, + ) + except ValueError as exc: + raise err(400, "bad_comment", str(exc)) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comment": comment} + + +@router.delete("/records/{pid}/comments/{comment_id}") +@router.delete("/customers/{pid}/comments/{comment_id}") +def remove_comment( + pid: int, + comment_id: str, + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + deleted = record_comments.delete_comment( + session.runtime, + pid, + comment_id, + session.uname, + admin=session.admin, + scope=scope, + ) + except record_comments.CommentForbidden: + raise err(403, "comment_forbidden", "only the author may delete this comment") + except record_comments.CommentsUnavailable: + raise _unavailable() + if not deleted: + raise err(404, "comment_not_found", "that comment no longer exists") + return {"ok": True, "id": comment_id} diff --git a/api/routes_script_views.py b/api/routes_script_views.py new file mode 100644 index 0000000000000000000000000000000000000000..3481df79475146575d2a5ae42ed2b05708dfd4c4 --- /dev/null +++ b/api/routes_script_views.py @@ -0,0 +1,329 @@ +"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). + +Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user +can build whatever they want through the Agent chat interface. be able to create any dashboard +they want. User should have the ability to see the code AND the dashboard output of course… Limit +the code script View per database… Any agent can add into more AI script, so we can see different +versions or different things the AI code for us."* + + GET /api/v1/script-views?database=K the views bound to ONE database + POST /api/v1/script-views create one {database, name?, source} + GET /api/v1/script-views/{id} one view, its source and its history + PUT /api/v1/script-views/{id} a NEW VERSION of the source + DELETE /api/v1/script-views/{id} drop it + POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} + +⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may +read ONLY the database it lives in, and a script that names another database is REFUSED with a +message naming both. *Not capped*: there is **no limit on how many script views a database may +carry**, because that is how an agent offers three attempts and the owner picks one. So nothing +below counts views. What IS bounded is what makes them big — one source is capped, and one view's +edit history is capped and REPORTS what it dropped. + +⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed +`session.user`, so a script written by an administrator and opened by a scoped analyst reads the +ANALYST's rows. The author decides what the code does; the reader decides what it can see. +⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script +that exfiltrates anything, because the only thing a script can return is a render spec drawn on +the screen of the person who ran it. There is no network, no file and no second reader. + +⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a +coroutine that would block the event loop for every other request in the container. FastAPI runs a +plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. +""" +import threading +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. +VIEWS_KEY = "script_views" + +MAX_NAME = 80 +MAX_SOURCE_BYTES = 128 * 1024 + +#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far +#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader +#: can see the history is partial instead of concluding the view was only ever saved twice. +MAX_HISTORY = 40 + +#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal +#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, +#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what +#: gives up. A 429 that says so is honest; an unbounded fork is not. +MAX_CONCURRENT_RUNS = 4 +_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this database has no script views", never to a 500 on the view rail.""" + try: + found = rt.get(VIEWS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _database_ok(session, database): + """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. + + ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True + for an ADMIN on any key at all, including one no database answers to. So a create validated + with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing + can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it + RAISES `UnknownTable`, which is exactly the question being asked. + """ + import core.perm_scope as perm_scope + try: + perm_scope.scoped_fields(session.user, database, st=session.runtime) + except perm_scope.UnknownTable: + raise err(404, "no_database", f"there is no database '{database}' in this workspace") + except perm_scope.Denied: + raise err(403, "forbidden", f"your account may not read '{database}'") + except perm_scope.Unresolvable as exc: + # The database is real and cannot be served under this call's constraints. Standing rule + # 1's second sentence: report the cause and the recommendation, never a bare refusal. + raise err(409, "unresolvable", str(exc)) from None + + +def _clean_source(raw): + source = str(raw or "") + if not source.strip(): + raise err(400, "no_source", "a script view needs some code") + if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + raise err(413, "source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") + return source + + +def _row(rec, *, source=False): + """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" + out = {"id": rec.get("id") or "", "database": rec.get("database") or "", + "name": rec.get("name") or "", "author": rec.get("author") or "", + "version": int(rec.get("version") or 1), + "created": rec.get("created") or "", "updated": rec.get("updated") or "", + "versions": len(rec.get("history") or []) + 1, + "trimmed": int(rec.get("trimmed") or 0)} + if source: + out["source"] = rec.get("source") or "" + out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", + "created": h.get("created") or "", + "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} + for h in reversed(rec.get("history") or []) if isinstance(h, dict)] + return out + + +def _limits(): + return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, + "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, + # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. + "maxViewsPerDatabase": None} + + +def _put(session, view_id, mutate): + """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" + def _set(cur): + cur = dict(cur or {}) + nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) + if nxt is None: + cur.pop(view_id, None) + else: + cur[view_id] = nxt + return cur + + session.runtime.update(VIEWS_KEY, _set, flush="sync") + + +def _mine_or_admin(session, rec): + """Who may EDIT or DELETE a view: its author, or an administrator. + + ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database + may run any view on it, under their OWN scope. That is the whole of "so we can see different + versions or different things the AI code for us": a colleague's attempt is worth nothing if + only its author can open it. + """ + if session.admin or str(rec.get("author") or "") == session.uname: + return + raise err(403, "forbidden", "only the author or an administrator can change this script view") + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/script-views") +def list_script_views(database: str = "", session: Session = Depends(require_session)): + """Every script view bound to ONE database, newest first. `database` is required.""" + key = str(database or "").strip() + if not key: + raise err(400, "no_database", "name the database whose script views you want") + _database_ok(session, key) + rows = [_row(rec) for rec in _all(session.runtime).values() + if isinstance(rec, dict) and str(rec.get("database") or "") == key] + rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) + return {"database": key, "views": rows, "limits": _limits()} + + +@router.post("/script-views") +def create_script_view(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. + + A script view that cannot be run is a broken feature the reader discovers by pressing a button, + and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, + so the refusal arrives while the author still has the code in front of them. + """ + import secrets # noqa: PLC0415 + import core.script_sandbox as sandbox # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "no_database", "a script view is bound to one database") + _database_ok(session, database) + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + view_id = "sv_" + secrets.token_urlsafe(9) + rec = {"id": view_id, "database": database, + "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], + "source": source, "author": session.uname, "version": 1, + "created": _now(), "updated": _now(), "history": [], "trimmed": 0} + _put(session, view_id, lambda _prior: rec) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell the author their + # script was saved when it was not. + raise err(503, "store_unavailable", "the script view was NOT created") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.get("/script-views/{view_id}") +def get_script_view(view_id: str, session: Session = Depends(require_session)): + """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _database_ok(session, str(rec.get("database") or "")) + return {"view": _row(rec, source=True), "limits": _limits()} + + +@router.put("/script-views/{view_id}") +def update_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + def _mutate(prior): + prior = dict(prior or rec) + history = list(prior.get("history") or []) + history.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", "created": prior.get("updated") or ""}) + dropped = max(0, len(history) - MAX_HISTORY) + prior["history"] = history[dropped:] if dropped else history + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + if body.get("name"): + prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the new version was NOT saved") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.delete("/script-views/{view_id}") +def delete_script_view(view_id: str, session: Session = Depends(require_session)): + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + _put(session, view_id, lambda _prior: None) + return {"deleted": view_id} + + +@router.post("/script-views/{view_id}/run") +def run_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. + + ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser + executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key + before this function ever sees it, so a renderer cannot be talked into running something by a + script that was itself perfectly well behaved. + + ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess + wait on the event loop is a ten-second outage for the whole container. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + database = str(rec.get("database") or "") + # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. + # It is checked exactly as a stored one is, because "unsaved" is not a permission. + draft = (body or {}).get("source") if isinstance(body, dict) else None + source = _clean_source(draft) if draft else str(rec.get("source") or "") + # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and + # asking twice builds a registry topic's pool twice. The three refusals are translated below + # instead, which is the same wall reached through the same door. + + if not _RUN_SLOTS.acquire(blocking=False): + raise err(429, "busy", + f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " + f"Try again in a moment") + try: + out = sandbox.run_view(session.user, database, source, st=session.runtime) + finally: + _RUN_SLOTS.release() + + # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" + # answered 200 would be a permission decision the client has to go looking for, and every + # other door in this app answers 403 for it. Everything BELOW this line is a well-formed + # request whose ANSWER is that the script did not produce a view — that is a 200 with + # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. + if out.get("code") == "unknown_table": + raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") + if out.get("code") == "denied": + raise err(403, "forbidden", out.get("message") or "your account may not read that database") + if out.get("code") == "unresolvable": + refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") + refusal.detail["error"]["limit"] = out.get("limit") or {} + raise refusal + + answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), + "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), + "ms": int(out.get("ms") or 0), "code": out.get("code") or "", + # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits + # were applied; on a Windows host the memory and CPU ones were not, and a screen + # that claims an enforcement which did not happen is the failure the rule is about. + "caps": out.get("caps") or {}} + if not answer["ok"]: + answer["error"] = out.get("message") or "the script view did not produce a view" + return answer diff --git a/api/routes_shares.py b/api/routes_shares.py index 0579302b5319d2a751a05174d98e94438b6f3b14..62278f5c62b12e8fbc3a8d25d03ffd9705fc8bd5 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1,393 +1,393 @@ -"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). - - GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} - PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) - GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} - -`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail -already speaks, now extended to folders and databases so there is ONE vocabulary in the UI -(R10: "the same picker views use"). - -⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires -`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's -CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could -widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor -for non-administrators; that is a courtesy, and this check is the wall. - -⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means -every account that can already open the surface: `require_session` plus the topic's own gate run -first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field -closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that -could already reach the data ([[aios-permissioning]]). - -⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED -MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` -and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be -DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes -`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded -by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every -column — and an `*` database grant admits every account in the tenant to all of it. -That is a real capability, deliberately kept; what was wrong was a docstring promising a second -wall that does not exist for this kind. Scoping user tables is booked, not done -(`waves/wave32/sharing-audit.md` S-8). - -⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS -registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` -— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here -whose object is invisible under that one is a row in a list that opens a refusal, which is what -made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but -the two vocabularies are still two. -""" -from fastapi import APIRouter, Body, Depends - -import core.shares as shares -import core.users as users -from deps import Session, err, require_session -# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES -# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the -# reader agreeing about one string is the whole difference between an Inbox row that opens the -# shared database and one that is quietly unclickable. -from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC - -router = APIRouter(prefix="/api/v1") - - -def _kind_or_400(raw): - try: - return shares._check_kind(raw) - except ValueError as e: - raise err(400, "bad_kind", str(e)) - - -# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── -# -# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all -# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free -# strings off the URL and the only dependency was `require_session`, so any signed-in account -# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, -# an object with no grant record skipped the check entirely and the caller was stamped OWNER — -# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not -# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. -# -# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own -# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the -# victim next opens the dialog. - -#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share -#: route is not told which topic — so resolving one means asking each. -_BUILTIN_TOPICS = ("customer", "product") - - -def _topics(session): - """Every topic whose workspace could hold a view or folder for this tenant. - - ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on - tenant #0) to answer a question about KEYS (D-185). - """ - try: - import core.user_tables as ut - return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) - except Exception: # noqa: BLE001 - return _BUILTIN_TOPICS - - -def _owns_object(session, kind, oid): - """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? - - ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking - each topic's workspace in turn, which is N store reads; making every share call pay that - would put a loop on a route the manage-access dialog opens. The dangerous path is the one - where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs - exactly there, and the common path (a record exists, `may_administer` decides) is untouched. - """ - if session.admin: - return True - if kind == "database": - # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already - # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" - # here would be the second definition this wave keeps finding. - try: - import core.user_tables as ut - return bool(ut.may_open(oid, session.uname, is_admin=session.admin, - st=session.runtime)) - except Exception: # noqa: BLE001 - return False - try: - import core.table_store as table_store - except Exception: # noqa: BLE001 - return False - for topic in _topics(session): - try: - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - except Exception: # noqa: BLE001 - continue - if hit: - # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the - # person whose personal stratum holds it — anybody else reaching this line is - # exactly the case S-1 describes. - return str(hit[0]) == str(session.uname) - return False - - -def _can_see_object(session, kind, oid): - """May this caller READ an object's grant list — i.e. can they reach the object at all? - - ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A - REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the - ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches - PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with - `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit - in the grid** — answered `404` when bob opened its manage-access dialog. Measured before - fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said - `404 no_object`. - ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", - and the wall consulted the grant registry (system A) plus stratum ownership, never the view's - `permissions` (system B) — which is the one that actually decides who may OPEN it. - ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not - gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — - 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a - legitimate collaborator their view does not exist. - - ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its - owner — that is S-1, and widening this predicate onto `put_share` would re-open it. - """ - if _owns_object(session, kind, oid): - return True - if kind != "view": - # A folder carries no per-object visibility flag of its own, and a database's `may_open` - # (inside `_owns_object`) already admits grantees. Nothing wider to ask. - return False - try: - import core.table_store as table_store - for topic in _topics(session): - hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) - if hit: - return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, - session.uname, is_admin=session.admin)) - except Exception: # noqa: BLE001 - return False - return False - - -def _entries_or_400(session, entries): - """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. - - ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: - a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE - of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, - so a typo'd name is stored, reported as a successful save, and never reaches anybody. The - sharer believes the person has access. That is item 18's plain reading. - ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker - (`_people`). One route, two populations, and the write door was the permissive one. - ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for - "every account that can already open the surface". - """ - known = {p["username"].strip().lower() for p in _people(session.tenant)} - unknown = [] - for e in entries or (): - if not isinstance(e, dict): - continue - user = str(e.get("user") or "").strip().lower() - if user and user != shares.EVERYONE and user not in known: - unknown.append(user) - if unknown: - raise err(400, "unknown_people", - "no active account in this workspace is named " - + ", ".join(sorted(set(unknown))) - + " — nothing was shared. Pick people from the list rather than typing a name.") - - -@router.get("/share/mine") -def my_shares(session: Session = Depends(require_session)): - """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). - - Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves - in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL - that is not malformed at all. - """ - return shares.shared_with(session.uname, st=session.runtime) - - -@router.get("/share/{kind}/{oid}") -def get_share(kind: str, oid: str, session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - rec = shares.grants(kind, oid, st=session.runtime) - role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) - may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime) - # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: - # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — - # to any signed-in session, about objects it cannot open. Now a caller with no role on an - # object must prove they can reach it, and gets a 404 otherwise: the same answer a - # non-existent id gives, so the route cannot be used to probe which ids are real. - # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a - # caller who has no relationship with the object at all. - if role is None and not _can_see_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - return { - **rec, - "role": role, - "mayAdminister": may_admin, - # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry - # them. `assignable_people` serves bare display names because `user`-kind CELLS store - # display names — that list's shape cannot change without migrating cell values — so - # this route serves objects of its own. Existing grants that were written as lowercased - # display names are normalised by the wave-21 cleanup script. - # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller - # who may open the editor. A read-only grantee gets the grant list (their fair question is - # "who else has this?") and not a directory of every account in the workspace. - "people": _people(session.tenant) if may_admin else [], - } - - -def _people(tenant): - """[{username, name}] for this tenant — same population as `assignable_people`, with the - BINDING identity alongside the display one.""" - try: - reg = users.registry() or {} - except Exception: - return [] - want = str(tenant or '').strip().lower() - out = [] - for uname, u in reg.items(): - if not isinstance(u, dict) or u.get('active') is False: - continue - if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: - continue - out.append({"username": str(uname), "name": str(u.get('name') or uname)}) - return sorted(out, key=lambda p: p["name"].lower()) - - -@router.put("/share/{kind}/{oid}") -def put_share(kind: str, oid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - body = body or {} - rec = shares.grants(kind, oid, st=session.runtime) - # An object with NO grant record yet has no owner — the first person to share it claims it. - # That is safe because reaching this route at all means passing the surface's own wall, and - # the alternative (refusing until somebody seeds an owner) would make a brand-new folder - # unshareable by the person who just made it. - if rec["owner"]: - if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime): - raise err(403, "not_owner", - "only the owner of this item (or an administrator) can change who it is " - "shared with") - # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record - # is still claimed by the first person to share it — that rule is right, and refusing until - # somebody seeds an owner would make a brand-new folder unshareable by the person who just - # made it. What was missing is the half the old comment ASSERTED and the code never did: the - # claimant has to be able to reach the object. Without this, any signed-in account could - # stamp itself owner of an id it had never seen and lock the real creator out for good. - elif not _owns_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - entries = body.get("entries") - if not isinstance(entries, list): - raise err(400, "bad_entries", - "entries must be a list of {user, role} — send [] to un-share, which is how " - "revoking is expressed") - _entries_or_400(session, entries) - out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, - st=session.runtime) - _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) - return out - - -def _notify_new_grantees(session, kind, oid, before, after): - """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. - - Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was - silent: the grant landed in a rail section the receiver had to notice on their own, which is - why "I shared it with you" and "I never saw it" were both true. - - ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read - because an alert is a live question about rows; a share is an EVENT that happened once, and - polling for it would mean re-deriving "was this new?" on every inbox open — the diff below - only exists here, at the moment the set changes. - - ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), - so every save re-sends everyone who was already there. Diffing against `before` is what stops - a rename or a role change from ringing the bell for people whose access did not change. - ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in - the tenant on a single click is a broadcast nobody asked for. The rail still shows it. - ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the - grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. - """ - try: - was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} - fresh = [str(e.get("user")) for e in (after or ()) - if isinstance(e, dict) and e.get("user") not in was - and e.get("user") != shares.EVERYONE] - if not fresh: - return - import core.alerts as alerts - - label, route, view_id = _object_ref(session, kind, oid) - if not route: - # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and - # `notification_view` would have to invent a target. Silence is the honest answer - # here; the rail still shows the grant under "Shared with me". - return - sharer = str(session.user.get("name") or session.uname) - for user in fresh: - # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must - # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes - # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED - # from there rather than typed again — one vocabulary, one owner. - # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY - # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and - # are honestly named by their machine; a SHARE has a real person, and only this call - # site knows who. ⛔ It is passed as its OWN field rather than recovered from the - # `detail` prose below: a sender parsed out of " shared this with you" breaks - # the first time the sentence is reworded, silently, in the header - # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. - alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, - detail=f"{sharer} shared this with you", actor=sharer, - st=session.runtime) - except Exception: # noqa: BLE001 - return - - -def _object_ref(session, kind, oid): - """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. - - ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT - WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced - `target: {module: "database", id: "view_42"}` — an instruction to open a database named - `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not - addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to - travel. Caught by looking at the notification the driver actually produced, not by reading - the code back. - - ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the - receiver nothing they can act on, and the id is already in the target. - ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than - a row that opens nowhere. - """ - try: - if kind == "database": - import core.user_tables as ut - defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} - # A user table IS its own route key in both vocabularies (`route_for_topic`). - return (str(defn.get("label") or "").strip() or "A database", str(oid), "") - import core.table_store as table_store - from routes_alerts import route_for_topic - for topic in _topics(session): - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - if not hit: - continue - route = route_for_topic(topic) - if not route: - break - row = hit[1] if len(hit) > 1 else {} - name = str((row or {}).get("name") or "").strip() - # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens - # the grid and stops there rather than naming a view the receiver did not get. - return (name or ("A view" if kind == "view" else "A folder"), - route, str(oid) if kind == "view" else "") - except Exception: # noqa: BLE001 - pass - return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "") +"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). + + GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} + PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) + GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} + +`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail +already speaks, now extended to folders and databases so there is ONE vocabulary in the UI +(R10: "the same picker views use"). + +⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires +`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's +CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could +widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor +for non-administrators; that is a courtesy, and this check is the wall. + +⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means +every account that can already open the surface: `require_session` plus the topic's own gate run +first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field +closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that +could already reach the data ([[aios-permissioning]]). + +⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED +MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` +and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be +DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes +`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded +by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every +column — and an `*` database grant admits every account in the tenant to all of it. +That is a real capability, deliberately kept; what was wrong was a docstring promising a second +wall that does not exist for this kind. Scoping user tables is booked, not done +(`waves/wave32/sharing-audit.md` S-8). + +⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS +registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` +— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here +whose object is invisible under that one is a row in a list that opens a refusal, which is what +made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but +the two vocabularies are still two. +""" +from fastapi import APIRouter, Body, Depends + +import core.shares as shares +import core.users as users +from deps import Session, err, require_session +# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES +# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the +# reader agreeing about one string is the whole difference between an Inbox row that opens the +# shared database and one that is quietly unclickable. +from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC + +router = APIRouter(prefix="/api/v1") + + +def _kind_or_400(raw): + try: + return shares._check_kind(raw) + except ValueError as e: + raise err(400, "bad_kind", str(e)) + + +# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── +# +# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all +# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free +# strings off the URL and the only dependency was `require_session`, so any signed-in account +# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, +# an object with no grant record skipped the check entirely and the caller was stamped OWNER — +# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not +# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. +# +# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own +# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the +# victim next opens the dialog. + +#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share +#: route is not told which topic — so resolving one means asking each. +_BUILTIN_TOPICS = ("customer", "product") + + +def _topics(session): + """Every topic whose workspace could hold a view or folder for this tenant. + + ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on + tenant #0) to answer a question about KEYS (D-185). + """ + try: + import core.user_tables as ut + return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) + except Exception: # noqa: BLE001 + return _BUILTIN_TOPICS + + +def _owns_object(session, kind, oid): + """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? + + ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking + each topic's workspace in turn, which is N store reads; making every share call pay that + would put a loop on a route the manage-access dialog opens. The dangerous path is the one + where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs + exactly there, and the common path (a record exists, `may_administer` decides) is untouched. + """ + if session.admin: + return True + if kind == "database": + # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already + # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" + # here would be the second definition this wave keeps finding. + try: + import core.user_tables as ut + return bool(ut.may_open(oid, session.uname, is_admin=session.admin, + st=session.runtime)) + except Exception: # noqa: BLE001 + return False + try: + import core.table_store as table_store + except Exception: # noqa: BLE001 + return False + for topic in _topics(session): + try: + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + except Exception: # noqa: BLE001 + continue + if hit: + # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the + # person whose personal stratum holds it — anybody else reaching this line is + # exactly the case S-1 describes. + return str(hit[0]) == str(session.uname) + return False + + +def _can_see_object(session, kind, oid): + """May this caller READ an object's grant list — i.e. can they reach the object at all? + + ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A + REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the + ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches + PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with + `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit + in the grid** — answered `404` when bob opened its manage-access dialog. Measured before + fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said + `404 no_object`. + ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", + and the wall consulted the grant registry (system A) plus stratum ownership, never the view's + `permissions` (system B) — which is the one that actually decides who may OPEN it. + ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not + gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — + 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a + legitimate collaborator their view does not exist. + + ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its + owner — that is S-1, and widening this predicate onto `put_share` would re-open it. + """ + if _owns_object(session, kind, oid): + return True + if kind != "view": + # A folder carries no per-object visibility flag of its own, and a database's `may_open` + # (inside `_owns_object`) already admits grantees. Nothing wider to ask. + return False + try: + import core.table_store as table_store + for topic in _topics(session): + hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) + if hit: + return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, + session.uname, is_admin=session.admin)) + except Exception: # noqa: BLE001 + return False + return False + + +def _entries_or_400(session, entries): + """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. + + ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: + a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE + of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, + so a typo'd name is stored, reported as a successful save, and never reaches anybody. The + sharer believes the person has access. That is item 18's plain reading. + ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker + (`_people`). One route, two populations, and the write door was the permissive one. + ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for + "every account that can already open the surface". + """ + known = {p["username"].strip().lower() for p in _people(session.tenant)} + unknown = [] + for e in entries or (): + if not isinstance(e, dict): + continue + user = str(e.get("user") or "").strip().lower() + if user and user != shares.EVERYONE and user not in known: + unknown.append(user) + if unknown: + raise err(400, "unknown_people", + "no active account in this workspace is named " + + ", ".join(sorted(set(unknown))) + + " — nothing was shared. Pick people from the list rather than typing a name.") + + +@router.get("/share/mine") +def my_shares(session: Session = Depends(require_session)): + """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). + + Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves + in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL + that is not malformed at all. + """ + return shares.shared_with(session.uname, st=session.runtime) + + +@router.get("/share/{kind}/{oid}") +def get_share(kind: str, oid: str, session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + rec = shares.grants(kind, oid, st=session.runtime) + role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime) + # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: + # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — + # to any signed-in session, about objects it cannot open. Now a caller with no role on an + # object must prove they can reach it, and gets a 404 otherwise: the same answer a + # non-existent id gives, so the route cannot be used to probe which ids are real. + # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a + # caller who has no relationship with the object at all. + if role is None and not _can_see_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + return { + **rec, + "role": role, + "mayAdminister": may_admin, + # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry + # them. `assignable_people` serves bare display names because `user`-kind CELLS store + # display names — that list's shape cannot change without migrating cell values — so + # this route serves objects of its own. Existing grants that were written as lowercased + # display names are normalised by the wave-21 cleanup script. + # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller + # who may open the editor. A read-only grantee gets the grant list (their fair question is + # "who else has this?") and not a directory of every account in the workspace. + "people": _people(session.tenant) if may_admin else [], + } + + +def _people(tenant): + """[{username, name}] for this tenant — same population as `assignable_people`, with the + BINDING identity alongside the display one.""" + try: + reg = users.registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + out = [] + for uname, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append({"username": str(uname), "name": str(u.get('name') or uname)}) + return sorted(out, key=lambda p: p["name"].lower()) + + +@router.put("/share/{kind}/{oid}") +def put_share(kind: str, oid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + body = body or {} + rec = shares.grants(kind, oid, st=session.runtime) + # An object with NO grant record yet has no owner — the first person to share it claims it. + # That is safe because reaching this route at all means passing the surface's own wall, and + # the alternative (refusing until somebody seeds an owner) would make a brand-new folder + # unshareable by the person who just made it. + if rec["owner"]: + if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime): + raise err(403, "not_owner", + "only the owner of this item (or an administrator) can change who it is " + "shared with") + # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record + # is still claimed by the first person to share it — that rule is right, and refusing until + # somebody seeds an owner would make a brand-new folder unshareable by the person who just + # made it. What was missing is the half the old comment ASSERTED and the code never did: the + # claimant has to be able to reach the object. Without this, any signed-in account could + # stamp itself owner of an id it had never seen and lock the real creator out for good. + elif not _owns_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + entries = body.get("entries") + if not isinstance(entries, list): + raise err(400, "bad_entries", + "entries must be a list of {user, role} — send [] to un-share, which is how " + "revoking is expressed") + _entries_or_400(session, entries) + out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, + st=session.runtime) + _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) + return out + + +def _notify_new_grantees(session, kind, oid, before, after): + """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. + + Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was + silent: the grant landed in a rail section the receiver had to notice on their own, which is + why "I shared it with you" and "I never saw it" were both true. + + ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read + because an alert is a live question about rows; a share is an EVENT that happened once, and + polling for it would mean re-deriving "was this new?" on every inbox open — the diff below + only exists here, at the moment the set changes. + + ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), + so every save re-sends everyone who was already there. Diffing against `before` is what stops + a rename or a role change from ringing the bell for people whose access did not change. + ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in + the tenant on a single click is a broadcast nobody asked for. The rail still shows it. + ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the + grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. + """ + try: + was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} + fresh = [str(e.get("user")) for e in (after or ()) + if isinstance(e, dict) and e.get("user") not in was + and e.get("user") != shares.EVERYONE] + if not fresh: + return + import core.alerts as alerts + + label, route, view_id = _object_ref(session, kind, oid) + if not route: + # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and + # `notification_view` would have to invent a target. Silence is the honest answer + # here; the rail still shows the grant under "Shared with me". + return + sharer = str(session.user.get("name") or session.uname) + for user in fresh: + # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must + # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes + # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED + # from there rather than typed again — one vocabulary, one owner. + # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY + # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and + # are honestly named by their machine; a SHARE has a real person, and only this call + # site knows who. ⛔ It is passed as its OWN field rather than recovered from the + # `detail` prose below: a sender parsed out of " shared this with you" breaks + # the first time the sentence is reworded, silently, in the header + # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. + alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, + detail=f"{sharer} shared this with you", actor=sharer, + st=session.runtime) + except Exception: # noqa: BLE001 + return + + +def _object_ref(session, kind, oid): + """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. + + ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT + WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced + `target: {module: "database", id: "view_42"}` — an instruction to open a database named + `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not + addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to + travel. Caught by looking at the notification the driver actually produced, not by reading + the code back. + + ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the + receiver nothing they can act on, and the id is already in the target. + ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than + a row that opens nowhere. + """ + try: + if kind == "database": + import core.user_tables as ut + defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} + # A user table IS its own route key in both vocabularies (`route_for_topic`). + return (str(defn.get("label") or "").strip() or "A database", str(oid), "") + import core.table_store as table_store + from routes_alerts import route_for_topic + for topic in _topics(session): + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + if not hit: + continue + route = route_for_topic(topic) + if not route: + break + row = hit[1] if len(hit) > 1 else {} + name = str((row or {}).get("name") or "").strip() + # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens + # the grid and stops there rather than naming a view the receiver did not get. + return (name or ("A view" if kind == "view" else "A folder"), + route, str(oid) if kind == "view" else "") + except Exception: # noqa: BLE001 + pass + return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "") diff --git a/api/routes_slack.py b/api/routes_slack.py index ce7c6c960ce3941d1a8cfdbb68ac4cc9342997b9..4cf017909fc13d069974d6d9cad3d927e8fd9fa5 100644 --- a/api/routes_slack.py +++ b/api/routes_slack.py @@ -229,7 +229,10 @@ def slack_creds(rt): def _summary(agent, modules): """One sentence per agent for the list row — computed here, because the alternative is one round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" - governed = [m for m in modules if m.get("enforced")] + # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to + # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and + # summarised every agent as "" (no databases at all). + governed = list(modules) perms = agent.get("perms") or {} open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) if not governed: @@ -304,16 +307,26 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): if not isinstance(a, dict): raise err(404, "no_such_agent", "no agent with that id") modules = routes_admin._perm_modules(session) - enforced = [m["key"] for m in modules if m.get("enforced")] + # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S + # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the + # key is None, so this list would have been EMPTY and the channel perms editor would have + # governed ZERO databases while looking entirely correct. A Slack channel is a principal of + # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. + governed = [m["key"] for m in modules] stored = a.get("perms") or {} principal = agent_principal(a) import core.perm_scope as perm_scope perms_out = {} - for k in enforced: + for k in governed: e = stored.get(k) + # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled + # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a + # channel agent is not a person with a share, so a `ut_*` key it has not been granted + # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the + # table routes would give it. perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_access(principal, k)), "filter": None, - "hiddenFields": []} + "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), + "filter": None, "hiddenFields": []} return {"id": str(agent_id), "channel": a.get("channel") or "", "channelName": a.get("channelName") or "", "label": a.get("label") or "", "active": a.get("active", True) is not False, @@ -326,7 +339,8 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): # the "Everything (admin)" state for a principal that cannot have it. "is_admin": False, "modules": modules, - "fields_by_module": {k: routes_admin._module_fields(k) for k in enforced}} + "fields_by_module": {k: routes_admin._module_fields(k, session=session) + for k in governed}} @router.put("/agents/{agent_id}/perms") @@ -341,14 +355,16 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), raise err(400, "empty_patch", "no perms to save") mods = routes_admin._perm_modules(session) # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot - # evaluate, refuses a hiddenFields key that names nothing, refuses an unenforced `ut_*` key, - # and refuses a BU condition the pushdown cannot read. Every one of those is exactly as true - # for a channel as for a person, and a second validator here is a second place for one of - # them to go missing. + # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the + # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, + # and a second validator here is a second place for one of them to go missing. + # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal + # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left + # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives + # it ([[two-gates-can-assert-opposite-things]]). cleaned = routes_admin._clean_perms( body.get("perms"), - enforced_keys={m["key"] for m in mods if m.get("enforced")}, - listed_keys={m["key"] for m in mods}) or {} + governed_keys={m["key"] for m in mods}, session=session) or {} import core.perm_scope as perm_scope diff --git a/api/routes_statements.py b/api/routes_statements.py index 771ac0a8a1b1326942145be58c132991bd3e66aa..c2cbedf8619c07e441bc5a6f1f80ffb5982b0772 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -129,9 +129,14 @@ def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): subject = (t.get("subject") or cs.DEFAULT_SUBJECT) try: subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) - except (KeyError, IndexError): + except Exception: # noqa: BLE001 # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they # can see what they typed rather than getting an opaque error. + # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. + # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a + # TypeError, and every one of them is the same user mistake this branch was written for. + # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the + # one screen somebody opens to avoid mailing 200 people the wrong thing. pass return { "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, @@ -153,7 +158,31 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)): names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] if not names: raise err(400, "bad_request", "name at least one customer") - override = (body.get("overrideTo") or "").strip() or None + # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A + # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent + # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) + # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On + # the one route in this product that is allowed to write to Odoo, the difference between a + # rehearsal and mailing a live customer was one empty string. + # + # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` + # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature + # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key + # being present, or `test: true`, is that statement, and an empty address beside it is the + # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` + # asserted with no key at all. + # + # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. + # `automationApi.testSendStatement` refuses an empty address before the request is built, and + # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client + # wall is bypassable by construction (anything can POST); this one is not. + declared_test = ("overrideTo" in body) or bool(body.get("test")) + override = str(body.get("overrideTo") or "").strip() + if declared_test and not override: + raise err(400, "no_override_address", + "a test send needs the address to send the test to. Without one this would " + "mail the customer's own address, which is the opposite of a test") + override = override or None if override and len(names) != 1: raise err(400, "bad_request", "a test send takes exactly one customer") t = body.get("templates") or {} diff --git a/api/routes_tables.py b/api/routes_tables.py index ed4db67e7a256237fb39d508b5dfcb7b8be6687e..4777831ff4e63d6fa73933a49a95d4abb8dcef9a 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -1,1623 +1,1879 @@ -"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). - -The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served -through the SAME grid machinery every other topic rides — `table_store` for the per-user -workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every -durable write except rows. Rows are the one genuinely new channel: the events seam has no row -event types (the user_tables docstring's `row_add` gate was described, never built), so row -add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + -`user_tables.may_open` — a connector-backed table can never accept an invented row. - -TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a -Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 -covers them for free. - -VISIBILITY is `user_tables.may_open` — creator or admin, fail-closed. There is no module grant -to check because a user table is not a module; the per-table wall is the whole wall, and it is -applied in `_defn_or_refuse` before any payload is built. -""" -import threading -import json -import time - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _ops(session, table_key): - import core.table_store as table_store - return table_store.make(f"{table_key}_table_workspace", st=session.runtime) - - -#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF -#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. -_REL_LOCK = threading.Lock() -_REL_DIRTY = {} -_REL_RUNNING = {} - - -def _refresh_relations(session): - """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. - - ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's - item 2 was "adding a new record visually takes too long, I need to be able to spam it", and - this function was the largest single cost inside `POST /tables/{key}/rows`: - `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can - decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something - does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the - default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit - before the 201 came back. - - ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty - whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam - the item asks us to support is exactly the load that would melt it. So this COALESCES: at - most one pass runs, and at most one is queued behind it. - - ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in - flight must still get a pass afterwards, because the in-flight one snapshotted before that - write existed. Without the flag the LAST add in a burst is precisely the one whose rollups - never update — the failure nobody would notice until a total was quietly wrong. - - Eventual consistency is the accepted trade and was already the documented posture: the tick - repairs materialised cells regardless, and answering 503 here would invite the browser to - repeat a mutation that already succeeded. - """ - tenant = str(getattr(session, "tenant", "") or "") - rt = session.runtime - with _REL_LOCK: - _REL_DIRTY[tenant] = True - if _REL_RUNNING.get(tenant): - return # a worker is live; it will see the flag and loop - _REL_RUNNING[tenant] = True - - def _worker(): - import automation_engine as engine - try: - while True: - with _REL_LOCK: - if not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - return - _REL_DIRTY.pop(tenant, None) - try: - engine.refresh_relations(rt, log=lambda *_args: None) - except Exception as exc: # noqa: BLE001 - # The source write already landed and the response is already sent. Log and - # let the tick repair it; never retry in a tight loop. - print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") - finally: - # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked - # RUNNING would never refresh again for the life of the process. - with _REL_LOCK: - if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - - threading.Thread(target=_worker, daemon=True, - name=f"rel-refresh:{tenant or 'default'}").start() - - -def _defn_or_refuse(session, table_key, st=None, defs_only=False): - """The per-table wall: 404 for a key that does not exist, 403 for one this session may not - open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but - not yours' is exactly what may_open is for. - - ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the - nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked - about the same table — it is simply not re-reading a 28.5 MB document to ask it. - - ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL - SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open - it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE - DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and - `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the - CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated - with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). - - ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, - `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass - it: a projected snapshot must never reach a post-write read-back (contract C5), and the - pre-write lend note below is the same argument one layer down. - - ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION - DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" - proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate - asserts the fast path was TAKEN. - """ - ut = _ut() - # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. - # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* - # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was - # the route count, not the caller count) and cost TWO whole-document deep copies at every one - # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions - # about ONE table. Lending here fixes every caller at once, including the eight routes the - # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · - # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). - # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The - # guard runs before any mutation and returns only the DEFINITION, so the snapshot never - # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes - # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which - # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it - # replaced. The remaining in-route reads are deliberately untouched and booked instead. - if st is None: - st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) - defn = ut.get(table_key, st=st) - if not defn: - raise err(404, "unknown_table", "that database does not exist") - if not ut.may_open(table_key, session.uname, session.admin, st=st): - raise err(403, "forbidden", "that database belongs to another user") - return defn - - -def _records_or_refuse(session, table_key, st=None): - """The human record-write wall for a database the automation engine owns.""" - # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — - # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. - st = st if st is not None else _ut().lend(session.runtime) - defn = _defn_or_refuse(session, table_key, st=st) - if not _ut().records_mutable(table_key, st=st): - raise err(403, "records_read_only", - "records in this automation-owned database are read-only. Add Instagram " - "handles in a Profile database and let enrichment populate this database") - return defn - - -#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT -#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. -#: -#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity -#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never -#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the -#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, -#: and the wall then hid the survivor from the person who just found it. They searched, they -#: paid, and the screen said nothing arrived. -#: -#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named -#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores -#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the -#: write target is whatever database the user points the Create-record action at. So the wall -#: governed a table nobody writes, and fixing EITHER half alone would have armed the other -#: ([[defects-that-mask-each-other]]). -#: -#: The register offered two exits and R4 already implied this one. Restoring per-user visibility -#: instead would have required R4's merge to stop crossing users — a bigger change, against the -#: ruling, to bring back a wall that never governed anything real. -#: -#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every -#: automation-written table has one (a scraped row says `automation`), so inference would hide -#: every scraped row from every non-admin — the same disappearance defect, one table wide instead -#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. - - -def _too_big(): - """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" - import routes_odoo_tables - return routes_odoo_tables.TooBigToMaterialise - - -def _read_through_rows(table_key, field_keys, rt=None): - """The mirror's rows for one read-through grid, projected to this table's declared columns. - - ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, - `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; - the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this - process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, - which is exactly the shape that satisfies the first wall while failing the second. - `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every - tenant, so the two walls are not substitutes for each other. - - ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which - owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror - through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's - by construction rather than by a second query that agrees today. A pid-only `SELECT` would be - cheaper and would also be a SECOND statement of what a row of this table is - ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, - never in how they ask. - - Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides - whether that is a 409 or an unresolved pid scope. - """ - import routes_odoo_tables - - rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} - for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] - rows_src.sort(key=lambda r: r["pid"]) - return rows_src - - -#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. -#: -#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= -#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the -#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. -#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a -#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). -#: -#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is -#: "membership is unresolved on this grid", which is a different claim and has to be made out -#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are -#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` -#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. -_PID_SCOPE_LIMIT = { - "subject": "pids", "effect": "unresolved", - "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " - "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " - "every total is a SQL count over the whole table", -} - - -def scoped_pool(session: Session, table_key: str, st=None): - """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. - - ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as - `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere - else, so the permission question is answered by the same code against the same document. - ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then - reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). - - `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a - caller that only needs "which rows of this database may this session touch" — record comments - — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second - idea of what a user table's pool is. - """ - defn = _defn_or_refuse(session, table_key, st=st) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - field_keys = {f["key"] for f in fields_base} - # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. - # - # This is the one function that turns "what is stored" into "what this session may see", which - # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events - # route, the comments wall and the assembly all reach rows through it, so they all convert - # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve - # an EMPTY GRID — correct-looking, wrong, and silent. - # ⚠ The wall above has already run. This adds no scope of its own and takes none away. - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) - except _too_big() as e: - # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. - # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not - # this): a caller that asked for every row of a 963,783-row grid cannot be served a - # short one. `scoped_pids` below takes the same refusal and answers a different - # question with it, because an envelope needs no row. - raise err(409, "window_required", str(e)) - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - # WAVE 27 / D-72: no per-row owner filter. The table-level wall above - # (`_defn_or_refuse` -> `may_open`) is the WHOLE wall — see the retirement note on - # PER_USER_TABLES' former home. A row this session can reach is a row the tenant owns. - r = {k: v for k, v in (row or {}).items() if k in field_keys} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - - -@router.patch("/tables/{table_key}/shared/{pid}") -def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has - been waiting for since it shipped (W29-T62, wave 30 T28). - - ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not - "sharing would be nice": a user-created column and its values live PER USER, so a shared view - filtering on one names a column other accounts do not have — and an unknown column is an - INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy - list would silently show the whole catalogue to everyone but its author. A column whose value - is the same for every reader is the precondition for editing it at all. - - ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no - reader and no writer by design; "may this session open this surface" is answered HERE, where - the session is. Do not push the question down there. - ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. - That is the point, and it is why this door declares the column too — a value with no - definition is a cell nobody can find. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("field") or "").strip() - if not key: - raise err(400, "bad_request", "a field key is required") - _defn_or_refuse(session, table_key) - from core import shared_overlay - if not shared_overlay.is_shared(table_key, key, st=session.runtime): - shared_overlay.put_field(table_key, key, { - "key": key, "label": str(body.get("label") or key), "source": "overlay", - "type": str(body.get("type") or "text"), "shared": True, - "createdBy": session.uname}, st=session.runtime) - try: - # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is - # the API that says so. It delegates to the plural, so both stay reachable through the one - # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 - # named it (the same lens that found `drop_field` had no door either). - stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), - st=session.runtime)} - except ValueError as e: - # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. - raise err(400, "bad_value", str(e)) - return {"ok": True, "pid": pid, "cells": stored, - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -@router.delete("/tables/{table_key}/shared/fields/{field_key}") -def delete_shared_field(table_key: str, field_key: str, - session: Session = Depends(require_session)): - """Remove a TENANT-WIDE column and every value in it. - - ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column - and none that removes one, so a column anybody added was permanent for the whole tenant. The - reachability gate found it from the other end — `shared_overlay.drop_field` was complete, - correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). - - ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a - value; dropping the column deletes that value for EVERY account at once, so it is the - destructive-op wall this repo already uses for a database delete — not `editRole`, which - governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). - ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed - is admin-only, which is the safe direction. - """ - _defn_or_refuse(session, table_key) - from core import shared_overlay - defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) - if not defn: - raise err(404, "unknown_field", "that column is not a shared column on this database") - owner = str(defn.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a tenant-wide column can be removed by its creator or an admin. This one " - f"was added by {owner or 'somebody else'}, and dropping it would delete the " - f"value for every account") - dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) - return {"ok": True, "dropped": bool(dropped), - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -def scoped_pids(session: Session, table_key: str, limits=None): - """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without - building a row. - - ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write - (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate - it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on - every toggle — and the six keys the events route actually reads from that assembly - (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. - `rows_src` was computed and discarded. - - ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: - `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the - row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the - middle. The row WALL is unchanged — a narrower or wider set here would be a permission - change, and this is a performance change. - - ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document - read, which is D-87 and W30-T31. This removes the row pass. - ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then - `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions - have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a - PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. - Booked because a stale performance note is how a wave re-fixes something twice - ([[stale-baseline-unreadable-deltas]]). - - ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a - list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a - read-through grid whose population exceeds one window). The pid set is then EMPTY, and every - consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the - silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a - write is expected to carry the sentence through. Omitting the list means the caller accepts an - unannounced empty scope, which is only ever right for a caller that does not use the pids. - """ - # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. - # - # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` - # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a - # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document - # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` - # off the definition — no row — so the WALL can be answered from the 0.1% projection. - # - # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED - # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. - # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads - # `readThrough` — a definition key, safe under the projection, and already lent the defn so it - # costs no read of its own. - defn = _defn_or_refuse(session, table_key, defs_only=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the - # comprehension below would answer an EMPTY pid set — and the promise this function makes is - # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror - # through the SAME fetch that function uses rather than growing a second idea of the row set; - # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, - rt=session.runtime) - return frozenset(r["pid"] for r in rows), fields_base, defn - except _too_big() as e: - # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning - # this into a 409 is what made both line grids unopenable: the envelope was refused - # over rows it never renders. The scope is empty and SAID to be empty. - if limits is not None: - limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) - return frozenset(), fields_base, defn - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection - # above cannot serve it and would raise rather than answer `{}` (that is the whole design of - # `_Projected`). The wall has already passed on the projected document, so this re-reads the - # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped - # answer to a question already answered, which is how two ideas of ownership got into this file - # once before (see `may_open`'s own note in `core/user_tables.py`). - # - # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection - # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the - # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten - # read-through databases (every one of them, which is why the switch was slow) skip the big - # read entirely. - whole = _ut().get(table_key, st=session.runtime) - if whole is None: - # Between the wall and here the table was deleted by another request. Same refusal the - # wall gives, rather than an empty pid set nobody can distinguish from an empty table. - raise err(404, "unknown_table", "that database does not exist") - pids = frozenset(int(rid) for rid in (whole.get("rows") or {}) if str(rid).isdigit()) - return pids, fields_base, whole - - -def ut_write_ctx(session: Session, table_key: str): - """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. - - Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this - or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller - that starts needing rows should fail on an empty list it can see, not on a KeyError. - """ - import aios_grid - from core import grid_events - - limits = [] - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(), - admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, - scope_key=table_key, table=_ops(session, table_key)) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", - fields_base=fields_base) - return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), - # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather - # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. - "limits": limits, "defn": defn} - - -def ut_assembly(session: Session, table_key: str, storage_key: str = "", - consume_corrections: bool = True, with_rows: bool = True, st=None): - """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so - `/workspace` and the events route consume any of the three interchangeably. - - Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is - customer-grain, so there is nothing to offer over user rows. - - ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets - cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its - own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and - wrong the moment the store learned about topics. - - ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the - caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from - `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: - * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over - 963,783 rows nobody was going to look at, which is D-174 in one sentence; - * every materialised `ut_*` database stops allocating a dict per row and sorting them on a - route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per - database switch (owner item 7). - ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers - the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved - on the write door. - """ - import aios_grid - from core import grid_events - - limits = [] - if with_rows: - # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only - # a READ route passes one — see `scoped_pool`'s own note and contract C5. - pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) - else: - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) - rows_src = [] - - ops = _ops(session, table_key) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(), - admin=session.admin, fallback_ws=None, seen_ids={}, - # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones - # that happen to carry a scoped `table`. - st=session.runtime, - scope_key=table_key, table=ops) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, - consume_corrections=consume_corrections) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, - storage_key=storage_key, fields_base=fields_base) - - return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, - # R9: the Cohorts column's cells from this table's own lists. - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), - # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test - # for is a key a consumer forgets to test for, and this one carries a refusal. - "limits": limits, "defn": defn} - - -def ut_label(defn, key, meta=None): - """THE name of a user table, resolved ONCE (wave 20, item 6a). - - `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that - shows a database name reads through here, because the alternative is what wave 20 found: the - rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker - showed the original (it reads the definition), and neither looked broken. - - ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver - stays because it makes every row already stored — renamed before that fix landed — read - correctly today, without a migration. - """ - return ((meta or {}).get(key, {}).get("name") - or (defn or {}).get("label") or key) - - -def nav_meta(session): - """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" - try: - got = session.runtime.get("nav_meta") - return got if isinstance(got, dict) else {} - except Exception: # noqa: BLE001 - return {} - - -@router.get("/tables") -def list_tables(session: Session = Depends(require_session)): - """This session's user tables — the list the '+ New database' surface renders. - - ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This - route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: - `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on - tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant - with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked - about every table; it is handed the document this function already holds. See - `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not - available. - """ - ut = _ut() - meta = nav_meta(session) - out = [] - tables = ut.all_tables(st=session.runtime) - lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) - for key, t in sorted(tables.items(), - key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): - if not ut.may_open(key, session.uname, session.admin, st=lent): - continue - out.append({"key": key, "label": ut_label(t, key, meta), - "source": t.get("source") or "Blank", - "recordsMutable": ut.records_mutable(key, st=lent), - "createdBy": t.get("createdBy") or "", - "created": t.get("created") or "", - "fields": [dict(f) for f in (t.get("fields") or [])], - "rowCount": len(t.get("rows") or {})}) - return {"tables": out} - - -#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a -#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* -#: -#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since -#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product -#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client -#: could make one, so the whole read-through path was reachable by editing Python — the -#: [[artifact-with-no-importer]] shape, twice burned in this repo already. -#: -#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and -#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that -#: topic can group by; a second list here would be a second definition of the same fact, and the -#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding -#: to a metric KEY instead of carrying SQL. -_ROLLUP_CACHE = {} - - -def _rollup_source_offer(): - """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. - - ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE - time refuses silently — the cells are simply left blank, hours later, on a column that looks - configured. Two exclusions do real work: - * a topic with NO dims cannot be grouped at all, so it can never key a parent row; - * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose - inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: - *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only - ever error. - Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what - the user is matching their own column against, and `payment_state` (a value) and - `partner` (an id) are matched to very different columns. - """ - if _ROLLUP_CACHE.get("offer"): - return _ROLLUP_CACHE["offer"] - from harness import semantic as sem - from harness import windows as W - ut = _ut() - - topics, metrics = sem.topics(), sem.metrics() - by_topic = {} - for key, m in metrics.items(): - # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. - if m.get("agg") in ("ratio", "derived"): - continue - by_topic.setdefault(m["topic"], []).append( - {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", - "description": m.get("description") or ""}) - - out = [] - for tkey, t in sorted(topics.items()): - dims = ((t.get("store") or {}).get("dims") or {}) - measures = by_topic.get(tkey) or [] - if not dims or not measures: - continue - out.append({ - "key": tkey, - "label": t.get("label") or tkey, - "grain": t.get("grain") or "", - "dims": [{"key": dkey, - "label": d.get("label") or dkey, - # `store_query` emits `_id` only when the dim carries a display name - # alongside the key; otherwise the value IS the key. `rollup_sql` handles - # both, and the editor says which so the user matches the right column. - "keyedBy": "id" if d.get("name_col") else "value"} - for dkey, d in dims.items()], - "measures": sorted(measures, key=lambda m: m["label"].lower()), - }) - - # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` - # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds - # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the - # validator refuses would let the editor build a field the save door rejects. - windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} - for k in ut.ROLLUP_SOURCE_WINDOWS] - offer = {"topics": out, "windows": windows} - _ROLLUP_CACHE["offer"] = offer - return offer - - -@router.get("/tables/rollup-sources") -def rollup_sources(session: Session = Depends(require_session)): - """The topic → metric → dim → window offer the rollup field editor renders. - - ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI - matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below - this line it still resolves; added ABOVE it, this endpoint would silently start arriving as - `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — - this is the cheap ordering that keeps it from mattering. - - ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model - files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live - Odoo data" in the field editor and been able to build a column that can only ever be blank, - because there is no mirror behind it. Three comments (here, in `apiBridge` and on the - `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, - and the mode switch's "render only when there is a choice" guard is built on that promise. - ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already - what `refresh` consults to decide whether these tables may exist at all, and a second copy of - the rule would be a second answer the day a tenant gains a mirror. - """ - import odoo_relational - if not odoo_relational.is_royal(session.tenant): - return {"topics": [], "windows": []} - return _rollup_source_offer() - - -@router.post("/tables", status_code=201) -def create_table(body: dict = Body(default=None), - session: Session = Depends(require_session)): - ut = _ut() - body = body or {} - label = str(body.get("label") or "").strip() - if not label: - raise err(400, "bad_label", "give the database a name") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was created") - source = body.get("source") - try: - key = ut.create(label, session.uname, fields=body.get("fields"), - source=source, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was created") - if not key: - raise err(400, "refused", - f"could not create it. The name may be empty or this tenant already has " - f"{ut.MAX_TABLES} databases") - return {"key": key} - - -@router.patch("/tables/{table_key}") -def patch_table(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Rename a database — IN ITS DEFINITION (wave 20, item 6a). - - ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the - nav's display layer; the definition is what the automation editor's database picker, the - schema drawer and every future reader see. A rename that lands in only one of them leaves a - picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an - amendment for whoever owns that door. - """ - defn = _defn_or_refuse(session, table_key) - ut = _ut() - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can rename it") - label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) - if not label: - raise err(400, "bad_label", "give the database a name") - return {"key": table_key, "label": label} - - -@router.get("/tables/{table_key}/footprint") -def table_footprint(table_key: str, session: Session = Depends(require_session)): - """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). - - Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories - without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled - like the delete itself: only someone who could delete may case the joint. - - ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row - COUNT needs the whole document, and only on a materialised table.""" - defn = _defn_or_refuse(session, table_key, defs_only=True) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - s = session.runtime - views, fields = set(), len(defn.get("fields") or []) - try: - bucket = s.get(f"{table_key}_table_workspace") or {} - for _u, ws in bucket.items(): - if isinstance(ws, dict): - views |= set((ws.get("views") or {}).keys()) - fields += len(ws.get("fields") or {}) # per-user custom/measure strata - except Exception: - pass - import core.shares as shares - g = shares.grants("database", table_key, st=s) - auto = [] - try: - import automation_engine as engine - for aid, d in (engine.all_definitions(s) or {}).items(): - if (d.get("config") or {}).get("targetTable") == str(table_key): - auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) - except Exception: - pass - # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where - # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, - # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the - # projection is not what makes it wrong. - # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 - # in a dialog headed "what dies with this database"). Booked rather than fixed here: this - # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` - # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. - if _ut().materialises(table_key, st=s, defn=defn): - rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) - else: - rows = 0 - return {"rows": rows, "fields": fields, "views": len(views), - "sharedUsers": len(g.get("entries") or []), - "automations": sorted(auto, key=lambda a: a["name"].lower())} - - -@router.delete("/tables/{table_key}") -def delete_table(table_key: str, session: Session = Depends(require_session)): - """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). - - ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the - creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role - grantee could reach this route and delete the database somebody shared with them. The wall - is now the definition's own `createdBy`, the same check the rename route always had. - - Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and - DISABLES bound automations with a status note — never deletes them. The client's confirm - dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the - dialog is a product requirement, not a formality.""" - defn = _defn_or_refuse(session, table_key) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - try: - import automation_engine as engine - engine.disable_for_table(session.runtime, table_key) - except Exception: - pass - try: - _ut().delete(table_key, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the delete did not land. Try again") - return {"ok": True} - - -#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. -_IG_FORWARDED = set() - - -def _ig_forward(session): - """Bring this tenant's Instagram tables onto the current schema, at most once per process. - - ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE - path. `ut_ensure` calling it is right for a schema an automation is about to append to, and - useless for a change a PERSON is waiting to see: the owner's report was *"the first field is - still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. - The write path stays exactly as it was — this is a second door to the same idempotent call, - not a replacement for it. - - ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per - tenant per process; `migrate_ig_tables` returns without a write when every table is already - current (the common case after the first read); and a failure is SWALLOWED — a migration must - never be the reason a database will not open. - - ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be - retried on every subsequent read of every table for the life of the process — the write path is - still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing - store call on the hot path of a grid that is trying to render. - """ - tenant = str(getattr(session, "tenant", "") or "") - if tenant in _IG_FORWARDED: - return - _IG_FORWARDED.add(tenant) - try: - import automation_engine as engine - engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") - - - -#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized -#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is -#: not — the shape this exists for is one already-paid provider payload per row. -JSON_LIST_MAX = 400 - - -def _thin_json(fields, merged, table_key=""): - """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. - - ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads - `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something - large rather than something empty. `_truncated` is what the viewer keys its fetch on. - - ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the - viewer needs no table key, no record id and no new props threaded down through three - components to find the document — the route that removed the value says where it went. One - writer of that address instead of a server rule and a client rule that must agree forever. - """ - json_keys = [str(f.get("key")) for f in (fields or []) - if str(f.get("type") or "") == "json"] - if not json_keys: - return merged - out = {} - for pid, cells in (merged or {}).items(): - row = cells - for key in json_keys: - raw = cells.get(key) - if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: - if row is cells: - row = dict(cells) - row[key] = json.dumps({ - "_truncated": True, "bytes": len(raw), - "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) - out[pid] = row - return out - - -@router.get("/tables/{table_key}/rows") -def table_rows(table_key: str, session: Session = Depends(require_session)): - """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, - identity}` — so the client's generic topic fetch consumes it with zero new parsing. - - ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell - from the OVERLAY stratum only (that is what makes a custom column render standalone) — a - user table's base values live in its DEFINITION rows, so they are layered UNDER the user's - overlay edits here: base first, overlay wins. Without this every base cell reads empty - (found by this route's own gate check, not by luck).""" - import aios_grid - - _ig_forward(session) - # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. - # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE - # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), - # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask - # about the SAME document in the SAME request. The lend is the fix W31 QA already built for - # `_defn_or_refuse`; this threads it through the three sites that never got it. - # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a - # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the - # value it replaced (contract C5). - _lent = _ut().lend(session.runtime) - g = ut_assembly(session, table_key, st=_lent) - # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge - # this line does not perform since the route was written (owner item 3, 2026-08-09: - # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). - # - # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` - # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table - # is overlay-typed by construction, so it looked up a key that could not be there and every - # such cell rendered blank. - # - # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds - # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed - # in the store exactly as they should. Only the READ-BACK dropped them, which is why the - # value survived the gesture, vanished on reload, and looked like "it reset itself" — and - # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. - # - # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition - # value must not shadow an edit the user has made on top of it. - # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this - # widens what a caller can SEE by exactly their own edits and nothing else. - _overlays = (g.get("ws") or {}).get("overlays") or {} - merged = {} - for _r in g["rows_src"]: - _pid = str(_r["pid"]) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _ov = _overlays.get(_pid) - if isinstance(_ov, dict): - _cells.update(_ov) - merged[_pid] = _cells - # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE - # JSON DOCUMENTS DO NOT RIDE THE LIST. - # - # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% - # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB - # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The - # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole - # vendor response crossed the wire, was parsed by the browser and held in memory purely so a - # clipped first line could be drawn. - # - # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by - # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — - # the one place a person actually reads it, for the one row they opened. The cell that rides - # the list is a VALID small document saying what it stands for, so the preview renders - # honestly instead of showing half a truncated brace. - # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, - # so a tenant using `json` for a short config sees no change at all. - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) - # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ - # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. - _stamp_ai_states(table_key, g["fields"], rows, st=_lent) - # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you - # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this - # database says so here, with its cause and the recommendation, rather than waiting to be - # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the - # honest answer for a table nothing limits — never an absent key, which a client cannot tell - # apart from an older server. - _report = _ut().limit_report(table_key, st=_lent) - # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the - # only thing that ever set this key; the write door never stopped working and every client - # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a - # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the - # customer scope — `grid_events.docs_for` — never a twin here. - from core import grid_events as _ge - return {"fields": g["fields"], "rows": rows, "today": g["today"], - "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, - admin=session.admin, st=session.runtime), - "pulled_at": time.strftime("%Y-%m-%d %H:%M"), - "identity": {"pid": "pid"}, - "scope": {"table": table_key, "rowCount": len(rows)}, - "limits": [_report] if _report else [], - "recordsMutable": _ut().records_mutable(table_key, st=_lent)} - - -#: The per-cell provenance a row carries on the wire, one key per enrichment column. -#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. -#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new -#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces -#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this -#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the -#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off -#: every field key, so no column can ever be called `_ai_*`. -AI_STATE_PREFIX = "_ai_" - - -def _stamp_ai_states(table_key, fields, rows, st=None): - """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. - - ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token - count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest - would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary - is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the - RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. - - ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no - enrichment column is untouched, and a freshly created column adds nothing until something - runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row - instead of against whatever was true when the value was written. - """ - cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] - if not cols: - return rows - import ai_enrich as _ae - for field in cols: - col = str(field.get("key") or "") - marks = _ut().ai_enrich_marks(table_key, col, st=st) - cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} - for row in (rows or []): - if not isinstance(row, dict): - continue - state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) - if state != "empty": - row[AI_STATE_PREFIX + col] = state - return rows - - -@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") -def table_cell(table_key: str, pid: str, fkey: str, - session: Session = Depends(require_session)): - """ONE cell, whole — the other half of `_thin_json`. - - ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor - for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON - viewer opens this for the one row a person is actually reading. - - ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's - view of the table), so this cannot become a side door onto a table the caller may not open — - which is the failure a "just fetch the raw cell" helper invites. - ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must - read back what they typed, not the definition value underneath it. - """ - g = ut_assembly(session, table_key) - field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) - if field is None: - raise err(404, "unknown field", f"{fkey!r} is not a column on this database") - row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) - if row is None: - raise err(404, "unknown record", f"no record {pid!r} in this database") - overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} - value = overlay.get(fkey, row.get(fkey)) - return {"table": table_key, "pid": str(pid), "field": str(fkey), - "value": "" if value is None else str(value)} - - -@router.post("/tables/{table_key}/rows", status_code=201) -def add_row(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). - - ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo - that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the - response rather than assume; the client re-anchors on what came back. - """ - _records_or_refuse(session, table_key) - ut = _ut() - values = (body or {}).get("values") or {} - if not isinstance(values, dict): - raise err(400, "bad_values", "values must be an object of {fieldKey: value}") - try: - rid = ut.add_row(table_key, values, session.uname, st=session.runtime, - rid=(body or {}).get("rid")) - except Exception: - raise err(503, "store_unavailable", "the row was not saved. The store refused") - if rid is None: - # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap - # sentence alone would misdirect — the reader would go and count rows. Ask the same - # validator the law used rather than re-deciding here (one rule, two voices). - pf = ut.profile_field(table_key, st=session.runtime) - if pf and pf["key"] in values: - _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) - if not ok: - raise err(400, "refused", - f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " - f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " - f"profile link (instagram.com/name)") - raise err(400, "refused", - f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") - _refresh_relations(session) - return {"rid": rid, "pid": int(rid)} - - -@router.post("/tables/{table_key}/rows/import", status_code=201) -def import_rows(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. - - Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a - spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, - nothing is matched or overwritten, and the dialog says so before the button is pressed. - - ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate - the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in - an import is not a stray to be tidied away — it means the client offered a target it should - not have, and silently dropping it would leave the user looking for a column of values that - never arrived. The refusal names the column. - - ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every - profile cell validates, because a half-imported file is the worst outcome available: the user - cannot tell which rows landed without reconciling the spreadsheet by hand. - """ - _records_or_refuse(session, table_key) - ut = _ut() - rows_in = (body or {}).get("rows") - if not isinstance(rows_in, list) or not rows_in: - raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") - if any(not isinstance(r, dict) for r in rows_in): - raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") - defn = _defn_or_refuse(session, table_key) - by_key = {f["key"]: f for f in (defn.get("fields") or [])} - asked = {k for r in rows_in for k in r} - unknown = sorted(k for k in asked if k not in by_key) - if unknown: - raise err(400, "unknown_field", - f"this database has no column {unknown[0]!r}") - computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) - if computed: - label = by_key[computed[0]].get("label") or computed[0] - raise err(400, "computed_field", - f"{label!r} is worked out from other columns, so it cannot be imported into") - # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. - # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a - # second client, or a future importer met no wall at all and the string landed verbatim in a - # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE - # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every - # rule on this route exists to prevent. The sentence names the row and the column, because - # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. - for index, row in enumerate(rows_in): - for key, value in row.items(): - why = ut.cell_type_refusal(by_key[key], value) - if why: - raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") - try: - made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "nothing was imported. The store refused") - if made is None: - raise err(400, "refused", - f"nothing was imported. {len(rows_in)} rows would take this database past " - f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") - _refresh_relations(session) - return {"imported": len(made), "pids": [int(r) for r in made]} - - -# --------------------------------------------------------------------------------------------- -# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) -# --------------------------------------------------------------------------------------------- -# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same -# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's -# definition to everyone without handing over the table. This supersedes wave 17's "fields are -# per-user" law for this path only; the connector scopes keep their own model. -# -# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the -# per-user workspace overlay, which is why the automation editor's "Automation column" picker -# could not see a column the user had just created — it reads the DEFINITION. Same defect shape -# as the rename (item 6a): two places to look, and the surfaces disagreed silently. - -def _field_or_refuse(session, table_key, fkey=""): - """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. - - ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and - `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they - write through the `user_tables` write doors), so no projected snapshot survives into a write. - ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 - draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" - defn = _defn_or_refuse(session, table_key, defs_only=True) - ut = _ut() - if not ut.is_user_table(table_key, st=session.runtime): - raise err(400, "not_a_user_table", - "only a user-created database has an editable schema. A connected source " - "owns its own columns") - if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, - st=session.runtime): - field = next((f for f in (defn.get("fields") or []) - if f.get("key") == str(fkey)), None) - if isinstance((field or {}).get("automation"), dict) \ - and field["automation"].get("preset") is True: - # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not - # ut.may_edit_field` above is the wall). Said out loud because the sentence itself - # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a - # blanket "pre-set fields are locked" would now be the server explaining a refusal - # it did not make — `preset_editable` is the one predicate that answers this. - raise err(403, "preset_field_locked", - "this is a pre-set column, so its name and type are fixed; you may sort, " - "filter or hide it, edit any Rollup column, and add your own columns") - raise err(403, "forbidden", "that column can only be changed by the database's creator " - "or an admin") - if not fkey and not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can add a column") - return defn - - -@router.post("/tables/{table_key}/fields", status_code=201) -def add_field(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - _field_or_refuse(session, table_key) - ut = _ut() - field = ut.add_field(table_key, body or {}, st=session.runtime) - if not field: - # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers - # None for every refusal, so this route said "check the name and type" to somebody whose - # name and type were fine and whose automation column named a flow that does not exist. - # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the - # one thing that was never wrong. Checked HERE, in the route's own words, because the - # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never - # re-implements it (a second copy of the rule is how two doors start disagreeing). - raise err(400, "refused", - _refusal_sentence(ut, session, body or {}, table_key=table_key)) - if field.get("type") == "link": - synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) - field = synced.get("field") or field - # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route - # became reachable for one. It was gated on `link` alone, while `patch_field` and - # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first - # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). - # The user would have created the column, watched a 201 come back, and read a blank cell for - # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. - # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in - # the tenant, and adding a text column has nothing to fold. The condition is now "is this - # field relational", which is the question that was always meant. - if field.get("type") in ("link", "rollup"): - _refresh_relations(session) - return _with_dropped(ut, {"field": field}, body) - - -def _with_dropped(ut, out, body): - """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). - - ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag - cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and - `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the - defect. T51's contract is that an unknown config key is dropped **and named**, so the naming - rides the response beside the accepted field rather than inside the validator. - - ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to - ignore the key, which is how a report stops being read before it stops being true. - """ - dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) - if dropped: - out = dict(out) - out["dropped"] = dropped - return out - - -def _refusal_sentence(ut, session, body, table_key="", fkey=""): - """Why was this column refused? The specific reason when we can name one, the general list - otherwise — never a specific-sounding guess.""" - # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag - # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment - # column would otherwise be explained by the flow law -- "pick a flow, or make this an - # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user - # never touched. - if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": - bag = (body or {}).get("aiEnrich") - if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): - return ("an AI enrichment column needs a prompt. It is the only thing that can " - "produce a value here, so a column without one would stay empty forever") - # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is - # what the contract asks for and what makes it actionable — "at most one" sends the reader - # hunting through a 40-column schema for a flag they cannot see from the header. - if isinstance((body or {}).get("profile"), dict): - # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at - # once (an `int` profile column on a table that already has a profile column), and the - # TYPE is the one that is wrong about what the caller just sent — unconditionally, no - # matter what else is on the table. Answering "you already have one" to somebody whose - # real mistake was the column type sends them to fix the wrong thing, which is the - # misdirection D-46 closed one door over. - if str((body or {}).get("type") or "text").strip().lower() != "text": - return ("a profile column is a flag on an ordinary TEXT column. It validates what " - "is typed into it, which it can only do for text") - existing = ut.profile_field(table_key, st=session.runtime) if table_key else None - if existing and existing.get("key") != str(fkey): - return (f"this database already has a profile column: " - f"{existing.get('label') or existing.get('key')!r}. A database has at most " - f"one, so the automation knows which handle to enrich; edit that column, or " - f"take the flag off it first") - bag = (body or {}).get("automation") - if isinstance(bag, dict): - flow = str(bag.get("flowId") or "").strip() - if not flow: - return ("an automation column has to name the automation that fills it. Pick a " - "flow, or make this an ordinary column") - if not ut.flow_bound(bag, st=session.runtime): - return (f"this column names automation {flow!r}, which does not exist in this " - f"workspace. It may have been deleted; pick a flow that is still there") - kind = str((body or {}).get("type") or "").strip() - if kind and kind not in ut.UT_FIELD_TYPES: - return (f"{kind!r} is not a column type here (types: " - f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") - return (f"the column was refused. Check the name and type, or the table may be at its " - f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") - - -@router.patch("/tables/{table_key}/fields/{fkey}") -def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Edit one column's definition, and MIGRATE its values when options are renamed. - - ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: - [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed - Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and - every saved view that filtered on it. - """ - _field_or_refuse(session, table_key, fkey) - ut = _ut() - body = body or {} - migrated = None - renames = body.get("renames") - if renames: - try: - migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the rename did not land. Try again") - # The per-user workspace strata and any view filter naming the old value are the OTHER - # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an - # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a - # missing counterpart must not lose the half that DID land. - try: - import core.table_store as table_store - fn = getattr(table_store, "rename_choice_values", None) - if callable(fn): - fn(table_key, fkey, renames, st=session.runtime) - migrated = dict(migrated or {}, workspace=True) - except Exception: # noqa: BLE001 - migrated = dict(migrated or {}, workspace=False) - field = ut.patch_field(table_key, fkey, body, st=session.runtime) - if not field: - # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as - # adding one, so it must get the same sentence naming the column that already holds the - # flag — a patch that answered "check the name and type" would send the reader to the - # one thing that was never wrong (the D-46 lesson, one door over). - raise err(400, "refused", - _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) - synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) - field = synced.get("field") or field - _refresh_relations(session) - out = {"field": field} - if migrated is not None: - out["migrated"] = migrated - return _with_dropped(ut, out, body) - - -def _fire_on_change(table_key, pid, changed, session): - """Run any `on_change` enrichment column whose prompt names a cell that just moved. - - ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. - `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a - 20,000-row import into 20,000 whole-document reads on the single process this product runs - (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and - takes the definition, so this reads once and asks about every column. - - ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; - an enrichment that could not run is a missing value, not a lost edit, and the run's own report - carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a - fan-out here would put a vendor call on the critical path of every keystroke-commit. - """ - import ai_enrich as _ae - - try: - defn = _ut().get(table_key, st=session.runtime) or {} - wanted = _ae.on_change_fields(defn, changed.keys()) - for field in wanted: - # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still - # somebody's edit spending somebody's tokens, so it is booked against the person who - # typed rather than left unattributed. - _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], - user=session.uname) - except Exception: # noqa: BLE001 - pass - - -@router.post("/tables/{table_key}/fields/{fkey}/enrich") -def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. - - ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides - (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the - tenant by opening a grid. - - `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the - value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic - plan, where the policy and the never-overwrite-a-human law both apply. The two are one - function with one flag, not two runners. - - ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by - reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a - ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a - populated `skipped` is a correct, informative answer, and the client must render it rather - than treat it as success. - """ - _field_or_refuse(session, table_key, fkey) - import ai_enrich as _ae - - rows = (body or {}).get("rows") - if rows is not None and not isinstance(rows, list): - raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " - "rows this column's own settings choose") - # The caller's permitted pool, the same one the row doors use. A named row outside it is - # dropped rather than refused: a stale client naming a record that has been deleted or moved - # out of scope should not fail a run over the rows it can legitimately fill. - if rows is not None: - allowed = {str(p) for p in scoped_pids(session, table_key)[0]} - rows = [str(r) for r in rows if str(r) in allowed] - # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" - # sends `always`. Anything else falls back to the column's own saved policy rather than to a - # default, so a typo cannot quietly widen what a run touches. - report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, - policy=str((body or {}).get("scope") or "") or None, - # A named row set through THIS door is a person asking. - manual=rows is not None, - # ⭐ W35-T41 / C7 — the usage ledger's attribution. - user=session.uname) - if report.get("problem"): - # A run that could not start at all is not a 200: nothing was attempted, nothing was - # spent, and the reason is actionable (no provider configured, or the wrong column). - raise err(400, "enrich_refused", str(report["problem"])) - return report - - -@router.delete("/tables/{table_key}/fields/{fkey}") -def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): - _field_or_refuse(session, table_key, fkey) - if not _ut().delete_field(table_key, fkey, st=session.runtime): - raise err(400, "refused", - "that column could not be removed. A database must keep at least one") - _refresh_relations(session) - return {"deleted": fkey} - - -@router.delete("/tables/{table_key}/rows/{rid}") -def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): - # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. - # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* - # A single DELETE was FIVE whole-document deep copies before the commit even began — three in - # the guard (`get` + `may_open` + `records_mutable`) and two more inside - # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads - # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** - # (D-185), so the guard alone was seconds of copying to answer questions about one row. - # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` - # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document - # under the store lock, so a lent snapshot can never be the thing written back. - # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually - # consistent delete is indistinguishable from one that did not work. This makes the guard - # cheap; it does not make the commit optimistic. - lent = _ut().lend(session.runtime) - _records_or_refuse(session, table_key, st=lent) - try: - ok = _ut().delete_row(table_key, rid, st=lent) - except Exception: - raise err(503, "store_unavailable", "the delete did not land. Try again") - if not ok: - raise err(400, "refused", "rows can only be deleted from user-created databases") - _refresh_relations(session) - return {"ok": True} - - -@router.patch("/tables/{table_key}/rows/{pid}") -def patch_row(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Cell edits — the products PATCH on the user-table ctx. Routed through - `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; - the accepted values are read BACK from the bucket, never echoed from the request.""" - from core import grid_events - - updates = dict(body or {}) - if not updates: - raise err(400, "empty_patch", "no fields to update") - _records_or_refuse(session, table_key) - g = ut_assembly(session, table_key, consume_corrections=False) - if pid not in g["pids"]: - raise err(403, "out_of_scope", "that row is not in this database") - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], - admin=session.admin, fallback_ws=None, seen_ids={}, hidden_keys=frozenset(), - st=session.runtime, # R6b (D-16) - scope_key=table_key, table=_ops(session, table_key)) - try: - grid_events.handle_one( - {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", - "pid": pid, "updates": updates}, ctx) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable. Your change was not saved") - # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. - # - # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH - # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. - # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's - # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began - # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` - # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring - # claiming otherwise is what makes the next reader look for a merge bug that is not here. - # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY - # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` - # asks whether THIS write landed, and this write goes to the definition. - stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) - .get(str(pid)) or {}) - accepted = {k: stored.get(k) for k in updates if k in stored} - - def _took(k): - """Did the cell TAKE this write? Normally that is "stored == asked". - - ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` - and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell - the client to roll back a write that landed. But it cannot simply be exempted either — - a junk handle leaves the OLD value sitting in `stored`, which would then read as - accepted. So the question asked is the exact one: **is what is stored the canonical form - of what was asked?** Anything else is a genuine refusal. - """ - if k not in accepted: - return False - want = str(updates[k]) - if stored.get(k) == want: - return True - pf = _ut().profile_field(table_key, st=session.runtime) - if pf and pf["key"] == k: - handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) - return bool(ok) and stored.get(k) == handle - return False - - refused = sorted(k for k in updates if not _took(k)) - # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write - # this cell?" is not recoverable from the value afterwards, so the only place to record it is - # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic - # run overwrite it, whatever the column's `overwrite` policy says. - # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused - # write `human` would freeze a cell against the agent on the strength of an edit that never - # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. - took = {k: v for k, v in accepted.items() if k not in refused} - if took: - try: - _ut().note_human_edit(table_key, took, pid, st=session.runtime) - except Exception: # noqa: BLE001 - # Provenance is metadata about a write that has already succeeded. Failing the - # request here would tell the user their edit was lost when it was not. - pass - _fire_on_change(table_key, pid, took, session) - out = {"pid": pid, "updates": accepted} - if refused: - out["refused"] = refused - # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a - # profile blank cleared. Without this the grid keeps painting a stale follower count under - # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other - # half: the client may keep only what the server actually took, and must be TOLD what else - # moved. Derived by diffing this row against what was asked for, so it cannot drift from - # whatever the clear rule decides to touch. - also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} - cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) - if cleared: - out["cleared"] = cleared - # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, - # amendment A5; SESSION B built and gated the function and correctly declared it INERT until - # this line existed, citing [[flag-shipped-without-its-writer]]). - # - # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead - # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the - # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we - # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try - # it again, the account is back". Without this call that person has no way back at all, and - # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. - # - # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this - # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a - # human touched the handle cell. - # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- - # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must - # be readable in the log rather than degrade into "the re-arm quietly stopped working". - _pf = _ut().profile_field(table_key, st=session.runtime) - if _pf and _pf["key"] in updates: - try: - import automation_engine as _engine - _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") - _refresh_relations(session) - return out +"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). + +The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served +through the SAME grid machinery every other topic rides — `table_store` for the per-user +workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every +durable write except rows. Rows are the one genuinely new channel: the events seam has no row +event types (the user_tables docstring's `row_add` gate was described, never built), so row +add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + +`user_tables.may_open` — a connector-backed table can never accept an invented row. + +TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a +Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 +covers them for free. + +VISIBILITY is `user_tables.may_open` — creator, admin, or a `core.shares` grant, fail-closed, +applied in `_defn_or_refuse` before any payload is built. + +⭐⭐ WAVE 36 (W36-T21 / OWNER RULING R6) — AND IT IS NO LONGER THE WHOLE WALL, WHICH IS THE POINT +OF THE TICKET. That paragraph used to end *"the per-table wall is the whole wall"*, and it was +true: a `ut_*` database was a BINARY door, so an admin could hand somebody all 31,418 rows of +`ut_odoo_invoices` or none of them, while `customer_data` had per-user row filters and hidden +fields. `perms.py`'s own docstring booked the fix and warned what half a fix looks like — *"a +stored `ut_*` wall would be INERT: the editor would say DENY, the table routes would keep serving, +and nothing anywhere would say so."* + +So THREE things are now true of every row this file serves, and each has one place: + * `perm_scope.may_read` — IF: admin, then an explicit stored `access: false`, then `may_open` + UNCHANGED. Composed, never merged. + * `scoped_pool`/`scoped_pids` — WHICH ROWS: the permanent filter, applied BEFORE `pids` is + taken, exactly where `routes_customers.grid_assembly` applies it. + * `ut_assembly` — WHICH FIELDS: the transitive hidden closure, applied AFTER + `workspace_wire`, on BOTH wires (the field list and the row payload). + +⛔ AND A DOOR THAT CANNOT APPLY ANY OF IT REFUSES RATHER THAN SERVING THE LOT — `_defn_or_refuse`'s +`scope_applied` flag, which defaults to fail-closed precisely because the one caller outside this +file cannot pass it. +""" +import threading +import json +import time + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _ops(session, table_key, st=None): + """This database's per-user workspace store, bound to the tenant. + + ⭐⭐ W36-T24 / D-214 — `st` LETS ONE ASSEMBLY LEND ITS OWN SNAPSHOT, AND THE DOCUMENT IT SAVES + IS `object_shares`, NOT THE WORKSPACE. MEASURED with a call-counting probe over `ut_assembly` + (D-214's own exit condition): one assembly took **5 whole-document reads for an admin and 6 + for a shared, row-scoped user** — `object_shares` TWICE (three times for a non-creator), + `user_tables` once, and `_table_workspace` twice. + + The `object_shares` repeats come from `grid_events._granted_views` and `_granted_folders`, + which ask `shares.shared_with` for the `view` and the `folder` kind separately, each reaching + the store through `tops.st` — this handle. `user_tables.lend` already memoises exactly those + two buckets for one pass (`_LENDABLE`), so handing the ops object a lend collapses them + without touching `core/shares.py`'s semantics or `grid_events`, which is in no wave-36 fence. + + ⛔ WHY IT IS SAFE ON A PATH THAT ALSO WRITES. `_Lent` serves ONLY `user_tables` and + `object_shares`; `_table_workspace` is not lendable, so every workspace read and write + passes straight through to the runtime, and `__getattr__` forwards `update` regardless. The + assembly performs no `object_shares` write, and `patch_row`'s post-write read-back reads + `session.runtime` directly rather than this handle — contract C5's rule, unbroken. + """ + import core.table_store as table_store + return table_store.make(f"{table_key}_table_workspace", + st=st if st is not None else session.runtime) + + +#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF +#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. +_REL_LOCK = threading.Lock() +_REL_DIRTY = {} +_REL_RUNNING = {} + + +def _refresh_relations(session): + """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. + + ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's + item 2 was "adding a new record visually takes too long, I need to be able to spam it", and + this function was the largest single cost inside `POST /tables/{key}/rows`: + `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can + decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something + does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the + default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit + before the 201 came back. + + ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty + whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam + the item asks us to support is exactly the load that would melt it. So this COALESCES: at + most one pass runs, and at most one is queued behind it. + + ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in + flight must still get a pass afterwards, because the in-flight one snapshotted before that + write existed. Without the flag the LAST add in a burst is precisely the one whose rollups + never update — the failure nobody would notice until a total was quietly wrong. + + Eventual consistency is the accepted trade and was already the documented posture: the tick + repairs materialised cells regardless, and answering 503 here would invite the browser to + repeat a mutation that already succeeded. + """ + tenant = str(getattr(session, "tenant", "") or "") + rt = session.runtime + with _REL_LOCK: + _REL_DIRTY[tenant] = True + if _REL_RUNNING.get(tenant): + return # a worker is live; it will see the flag and loop + _REL_RUNNING[tenant] = True + + def _worker(): + import automation_engine as engine + try: + while True: + with _REL_LOCK: + if not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + return + _REL_DIRTY.pop(tenant, None) + try: + engine.refresh_relations(rt, log=lambda *_args: None) + except Exception as exc: # noqa: BLE001 + # The source write already landed and the response is already sent. Log and + # let the tick repair it; never retry in a tight loop. + print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") + finally: + # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked + # RUNNING would never refresh again for the life of the process. + with _REL_LOCK: + if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + + threading.Thread(target=_worker, daemon=True, + name=f"rel-refresh:{tenant or 'default'}").start() + + +def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=False): + """The per-table wall: 404 for a key that does not exist, 403 for one this session may not + open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but + not yours' is exactly what may_open is for. + + ⭐⭐ W36-T21 / R6 / CONTRACT C1 — `scope_applied` IS THE HALF THAT MAKES THE WALL NON-INERT, + AND IT DEFAULTS TO FAIL-CLOSED ON PURPOSE. + + `perms.py`'s own docstring described exactly the defect this parameter prevents: *"a stored + `ut_*` wall would be INERT — the editor would say DENY, the table routes would keep serving, + and nothing anywhere would say so."* So a door that will apply C1's row/field wall to what it + is about to serve says so HERE, in writing, and a door that will not is REFUSED for any + principal carrying a wall on this database (`perm_scope.wall_declared`). + + ⛔ THE DEFAULT IS `False` BECAUSE THE ONE CALLER OUTSIDE THIS FILE CANNOT PASS IT. + `routes_odoo_tables`' windowed rows route calls this guard and then builds its own SQL — it + has no way to apply a row filter or a hidden-field closure, and it is in no wave-36 fence. An + opt-OUT default would have left that door silently serving a walled account the whole table, + which is the INERT wall by another route. Opting IN means a door added tomorrow is refused + until somebody has thought about it ([[default-must-pass-its-own-guard]]). + + ⚠ AND IT COSTS NOTHING FOR EVERYBODY WITH NO WALL — which today is every account in every + tenant, because no `ut_*` wall has ever been storable. `wall_declared` is False for an admin + and False for a record with no entry, so this branch cannot fire until an administrator + deliberately stores one. + + ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the + nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked + about the same table — it is simply not re-reading a 28.5 MB document to ask it. + + ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL + SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open + it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE + DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and + `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the + CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated + with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). + + ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, + `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass + it: a projected snapshot must never reach a post-write read-back (contract C5), and the + pre-write lend note below is the same argument one layer down. + + ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION + DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" + proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate + asserts the fast path was TAKEN. + """ + ut = _ut() + # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. + # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* + # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was + # the route count, not the caller count) and cost TWO whole-document deep copies at every one + # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions + # about ONE table. Lending here fixes every caller at once, including the eight routes the + # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · + # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). + # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The + # guard runs before any mutation and returns only the DEFINITION, so the snapshot never + # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes + # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which + # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it + # replaced. The remaining in-route reads are deliberately untouched and booked instead. + if st is None: + st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) + defn = ut.get(table_key, st=st) + if not defn: + raise err(404, "unknown_table", "that database does not exist") + # ⭐⭐ W36-T21 — ONE evaluator for the IF question, and it CALLS `may_open` rather than + # replacing it. `perm_scope.may_read` is admin -> an explicit stored `access: false` -> + # `user_tables.may_open`, unmodified. Two questions stay two questions: `may_open` still + # decides IF the database is visible, C1 decides WHICH rows and fields. + import core.perm_scope as perm_scope + if not perm_scope.may_read(session.user, table_key, st=st): + raise err(403, "forbidden", "that database belongs to another user") + if not scope_applied and perm_scope.wall_declared(session.user, table_key): + # R6's second sentence: a limit that cannot be met is a SENTENCE naming the cause and a + # fix, never a short answer — and here the short answer would be the WHOLE database. + raise err(409, "scope_not_applied", + "an administrator has restricted which rows and columns of this database you " + "may see, and this view cannot apply that restriction. Open the database from " + "the navigation, where the restriction is applied, or ask an administrator to " + "remove it") + return defn + + +def _records_or_refuse(session, table_key, st=None): + """The human record-write wall for a database the automation engine owns.""" + # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — + # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. + st = st if st is not None else _ut().lend(session.runtime) + # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a + # SCOPED pid set of its own: `patch_row` refuses a pid outside `ut_assembly`'s `pids`, and + # `delete_row` asks the same question just below. Refusing here instead would make a walled + # database READ-ONLY rather than row-scoped, which is a different product. + defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) + if not _ut().records_mutable(table_key, st=st): + raise err(403, "records_read_only", + "records in this automation-owned database are read-only. Add Instagram " + "handles in a Profile database and let enrichment populate this database") + return defn + + +#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT +#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. +#: +#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity +#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never +#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the +#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, +#: and the wall then hid the survivor from the person who just found it. They searched, they +#: paid, and the screen said nothing arrived. +#: +#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named +#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores +#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the +#: write target is whatever database the user points the Create-record action at. So the wall +#: governed a table nobody writes, and fixing EITHER half alone would have armed the other +#: ([[defects-that-mask-each-other]]). +#: +#: The register offered two exits and R4 already implied this one. Restoring per-user visibility +#: instead would have required R4's merge to stop crossing users — a bigger change, against the +#: ruling, to bring back a wall that never governed anything real. +#: +#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every +#: automation-written table has one (a scraped row says `automation`), so inference would hide +#: every scraped row from every non-admin — the same disappearance defect, one table wide instead +#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. + + +def _too_big(): + """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" + import routes_odoo_tables + return routes_odoo_tables.TooBigToMaterialise + + +def _read_through_rows(table_key, field_keys, rt=None): + """The mirror's rows for one read-through grid, projected to this table's declared columns. + + ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, + `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; + the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this + process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, + which is exactly the shape that satisfies the first wall while failing the second. + `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every + tenant, so the two walls are not substitutes for each other. + + ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which + owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror + through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's + by construction rather than by a second query that agrees today. A pid-only `SELECT` would be + cheaper and would also be a SECOND statement of what a row of this table is + ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, + never in how they ask. + + Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides + whether that is a 409 or an unresolved pid scope. + """ + import routes_odoo_tables + + rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} + for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] + rows_src.sort(key=lambda r: r["pid"]) + return rows_src + + +#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. +#: +#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= +#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the +#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. +#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a +#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). +#: +#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is +#: "membership is unresolved on this grid", which is a different claim and has to be made out +#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are +#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` +#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. +_PID_SCOPE_LIMIT = { + "subject": "pids", "effect": "unresolved", + "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " + "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " + "every total is a SQL count over the whole table", +} + + +def scoped_pool(session: Session, table_key: str, st=None): + """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. + + ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as + `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere + else, so the permission question is answered by the same code against the same document. + ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then + reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). + + `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a + caller that only needs "which rows of this database may this session touch" — record comments + — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second + idea of what a user table's pool is. + + ⭐⭐ W36-T21 / R6 — THE ROW WALL RUNS HERE NOW, ON EVERY DATABASE, and the position is the + whole of it: BEFORE `pids` is taken. `routes_customers.grid_assembly` says the same thing in + the same words for `customer_data` — everything downstream is bounded by that frozenset + (`allowed_pids` for the workspace, cohort membership, every write door's pid check), so + scoping here means a row this account may not see never enters ANY of them, rather than being + filtered out of one payload and surviving in another. + + ⚠ THE FIELD HALF IS NOT HERE, and that is parity rather than an omission: the hidden closure + must cover the user's own `custom_`/`measure_` columns, which do not exist until + `workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does. + """ + defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + field_keys = {f["key"] for f in fields_base} + # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. + # + # This is the one function that turns "what is stored" into "what this session may see", which + # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events + # route, the comments wall and the assembly all reach rows through it, so they all convert + # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve + # an EMPTY GRID — correct-looking, wrong, and silent. + # ⚠ The wall above has already run. This adds no scope of its own and takes none away. + import core.perm_scope as perm_scope + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) + except _too_big() as e: + # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. + # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not + # this): a caller that asked for every row of a 963,783-row grid cannot be served a + # short one. `scoped_pids` below takes the same refusal and answers a different + # question with it, because an envelope needs no row. + raise err(409, "window_required", str(e)) + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + # WAVE 27 / D-72: no per-row OWNER filter — the row's creator is not a permission. What + # runs below is a different thing entirely: the permanent filter an ADMIN declared for + # this account (W36-T21 / R6), the same one `grid_assembly` has applied to `customer_data` + # since wave 15. A row this session can reach is a row the tenant owns AND the wall admits. + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + # ⭐⭐ W36-T21 — `permits()`, not `matches()`: an unanswerable permanent filter DENIES rather + # than being ignored. Evaluated against the DECLARED contract (`fields_base`), which is what + # `routes_admin._clean_perms` validates a stored filter against, so the two cannot disagree + # about what a column is. + rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + + +@router.patch("/tables/{table_key}/shared/{pid}") +def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has + been waiting for since it shipped (W29-T62, wave 30 T28). + + ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not + "sharing would be nice": a user-created column and its values live PER USER, so a shared view + filtering on one names a column other accounts do not have — and an unknown column is an + INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy + list would silently show the whole catalogue to everyone but its author. A column whose value + is the same for every reader is the precondition for editing it at all. + + ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no + reader and no writer by design; "may this session open this surface" is answered HERE, where + the session is. Do not push the question down there. + ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. + That is the point, and it is why this door declares the column too — a value with no + definition is a cell nobody can find. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("field") or "").strip() + if not key: + raise err(400, "bad_request", "a field key is required") + _defn_or_refuse(session, table_key) + from core import shared_overlay + if not shared_overlay.is_shared(table_key, key, st=session.runtime): + shared_overlay.put_field(table_key, key, { + "key": key, "label": str(body.get("label") or key), "source": "overlay", + "type": str(body.get("type") or "text"), "shared": True, + "createdBy": session.uname}, st=session.runtime) + try: + # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is + # the API that says so. It delegates to the plural, so both stay reachable through the one + # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 + # named it (the same lens that found `drop_field` had no door either). + stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), + st=session.runtime)} + except ValueError as e: + # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. + raise err(400, "bad_value", str(e)) + return {"ok": True, "pid": pid, "cells": stored, + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +@router.delete("/tables/{table_key}/shared/fields/{field_key}") +def delete_shared_field(table_key: str, field_key: str, + session: Session = Depends(require_session)): + """Remove a TENANT-WIDE column and every value in it. + + ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column + and none that removes one, so a column anybody added was permanent for the whole tenant. The + reachability gate found it from the other end — `shared_overlay.drop_field` was complete, + correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). + + ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a + value; dropping the column deletes that value for EVERY account at once, so it is the + destructive-op wall this repo already uses for a database delete — not `editRole`, which + governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). + ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed + is admin-only, which is the safe direction. + """ + _defn_or_refuse(session, table_key) + from core import shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) + if not defn: + raise err(404, "unknown_field", "that column is not a shared column on this database") + owner = str(defn.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a tenant-wide column can be removed by its creator or an admin. This one " + f"was added by {owner or 'somebody else'}, and dropping it would delete the " + f"value for every account") + dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) + return {"ok": True, "dropped": bool(dropped), + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +def scoped_pids(session: Session, table_key: str, limits=None, st=None): + """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without + building a row. + + ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write + (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate + it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on + every toggle — and the six keys the events route actually reads from that assembly + (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. + `rows_src` was computed and discarded. + + ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: + `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the + row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the + middle. The row WALL is unchanged — a narrower or wider set here would be a permission + change, and this is a performance change. + + ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document + read, which is D-87 and W30-T31. This removes the row pass. + ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then + `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions + have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a + PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. + Booked because a stale performance note is how a wave re-fixes something twice + ([[stale-baseline-unreadable-deltas]]). + + ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a + list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a + read-through grid whose population exceeds one window). The pid set is then EMPTY, and every + consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the + silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a + write is expected to carry the sentence through. Omitting the list means the caller accepts an + unannounced empty scope, which is only ever right for a caller that does not use the pids. + """ + # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. + # + # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` + # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a + # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document + # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` + # off the definition — no row — so the WALL can be answered from the 0.1% projection. + # + # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED + # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. + # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads + # `readThrough` — a definition key, safe under the projection, and already lent the defn so it + # costs no read of its own. + import core.perm_scope as perm_scope + # ⭐ W36-T24 / D-214 — `st` LETS THE ASSEMBLY LEND ITS OWN PASS, exactly as `scoped_pool` has + # since W33-T03. ⛔ It must be a PROJECTED lend (`lend_defs`), not a whole one: the saving + # D-213 bought on the database-switch path is that this wall answers from the 0.1% document, + # and handing it `lend()` would quietly take that back while looking like an optimisation. + defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the + # comprehension below would answer an EMPTY pid set — and the promise this function makes is + # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror + # through the SAME fetch that function uses rather than growing a second idea of the row set; + # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, + rt=session.runtime) + rows = perm_scope.apply_row_scope(rows, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows), fields_base, defn + except _too_big() as e: + # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning + # this into a 409 is what made both line grids unopenable: the envelope was refused + # over rows it never renders. The scope is empty and SAID to be empty. + if limits is not None: + limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) + return frozenset(), fields_base, defn + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection + # above cannot serve it and would raise rather than answer `{}` (that is the whole design of + # `_Projected`). The wall has already passed on the projected document, so this re-reads the + # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped + # answer to a question already answered, which is how two ideas of ownership got into this file + # once before (see `may_open`'s own note in `core/user_tables.py`). + # + # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection + # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the + # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten + # read-through databases (every one of them, which is why the switch was slow) skip the big + # read entirely. + whole = _ut().get(table_key, st=session.runtime) + if whole is None: + # Between the wall and here the table was deleted by another request. Same refusal the + # wall gives, rather than an empty pid set nobody can distinguish from an empty table. + raise err(404, "unknown_table", "that database does not exist") + # ⭐⭐ W36-T21 — AND THE ROW WALL, WHICH IS WHY THIS ARM CAN NO LONGER ALWAYS SKIP THE ROWS. + # + # ⛔ THE PROMISE THIS FUNCTION MAKES IS THAT ITS PID SET IS **IDENTICAL** TO `scoped_pool`'s, + # not merely cheaper. `scoped_pool` now narrows its rows by the permanent filter before taking + # pids, so a set built here from the raw row ids would be WIDER — and every consumer of these + # pids (the workspace envelope, cohort membership, `patch_row`'s scope check) would admit rows + # the read door refuses. That is two ideas of one row set, which is the exact defect class + # `may_open`'s own wave-20 note records ([[one-question-two-normalizers]]). + # + # ⭐ AND W30-T30's SAVING SURVIVES FOR EVERYBODY IT WAS FOR. `row_scope_applies` is False for + # an admin and for every record with no declared filter, which is every account in every + # tenant today — those callers take the id comprehension exactly as before and build no row. + # Only a principal an administrator has actually row-scoped pays the pass, and for them the + # alternative is not "cheaper" but "wrong". + rows = whole.get("rows") or {} + if not perm_scope.row_scope_applies(session.user, table_key): + return frozenset(int(rid) for rid in rows if str(rid).isdigit()), fields_base, whole + keys = {f["key"] for f in fields_base if f.get("key")} + scoped = perm_scope.apply_row_scope( + [{**{k: v for k, v in (row or {}).items() if k in keys}, "pid": int(rid)} + for rid, row in rows.items() if str(rid).isdigit()], + session.user, table_key, fields_base) + return frozenset(r["pid"] for r in scoped), fields_base, whole + + +def ut_write_ctx(session: Session, table_key: str): + """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. + + Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this + or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller + that starts needing rows should fail on an empty list it can see, not on a KeyError. + """ + import aios_grid + from core import grid_events + + limits = [] + # ⭐ W36-T24 / D-214 — ONE lend for the whole pass, so the wall and the grant legs stop + # reading `object_shares` once each. Projected, because this ctx reads no row either. + lent = _ut().lend_defs(session.runtime) + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=lent) + # ⭐⭐ W36-T21 — THE FIELD WALL ON THE **WRITE** CTX, and it is not a copy of the read one. + # `grid_events` refuses a hidden key by asking `ctx.hidden_keys` (`grid_events.py:1802` and + # `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully + # writable on the EVENTS transport — a wall on one wire and not the other is the shape + # `strip_row`'s own note warns about, with the sign flipped. + hidden = _ut_hidden(session, table_key, fields_base) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, + admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, + scope_key=table_key, table=_ops(session, table_key, st=lent)) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", + fields_base=fields_base) + fields, _rows, hidden = _ut_field_wall(session, table_key, fields, []) + return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "hidden": hidden, + "derived": aios_grid.cohort_cells(lists), + "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather + # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. + "limits": limits, "defn": defn} + + +def _ut_hidden(session, table_key, fields): + """The hidden-field closure for THIS session on THIS database — C1's field half, once. + + ⚠ Named rather than inlined at its four call sites for the reason `may_open`'s own note gives: + four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE + evaluator; this is just the `ut_*` caller's shorthand for it. + """ + import core.perm_scope as perm_scope + return perm_scope.hidden_keys(session.user, table_key, fields) + + +def _ut_field_wall(session, table_key, fields, rows_src): + """`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires. + + ⭐⭐ W36-T21 — the same three lines `routes_customers.grid_assembly` runs for `customer_data`, + in the same position: AFTER `workspace_wire`, because the closure must cover the user's own + `custom_` and `measure_` columns and those do not exist until it has run. + + ⛔ BOTH WIRES, ALWAYS. `strip_row`'s docstring is the record of why: the field LIST and the + ROW payload are two different wires, and narrowing only the first leaves the value sitting in + the second where anything can read it. A formula (or a rollup) over a hidden column comes out + too — hiding the input while shipping the dependent either leaks the input wearing a derived + column's name or computes a wrong one. + + ⚠ IT TAKES NO `hidden` ARGUMENT, deliberately. The base-level closure the write ctx computed + is a SUBSET of this one by construction — same evaluator, a strictly larger field list — so + accepting it would be a second input that can only ever be redundant, i.e. a parallel code + path with nothing to say ([[one-question-two-normalizers]]). + """ + import core.perm_scope as perm_scope + hide = perm_scope.hidden_keys(session.user, table_key, fields) + if not hide: + return fields, rows_src, frozenset() + fields = [f for f in fields if f.get("key") not in hide] + rows_src = [perm_scope.strip_row(r, hide) for r in (rows_src or [])] + return fields, rows_src, hide + + +def ut_assembly(session: Session, table_key: str, storage_key: str = "", + consume_corrections: bool = True, with_rows: bool = True, st=None): + """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so + `/workspace` and the events route consume any of the three interchangeably. + + Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is + customer-grain, so there is nothing to offer over user rows. + + ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets + cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its + own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and + wrong the moment the store learned about topics. + + ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the + caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from + `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: + * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over + 963,783 rows nobody was going to look at, which is D-174 in one sentence; + * every materialised `ut_*` database stops allocating a dict per row and sorting them on a + route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per + database switch (owner item 7). + ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers + the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved + on the write door. + """ + import aios_grid + from core import grid_events + + limits = [] + # ⭐⭐ W36-T24 / D-214 — **ONE LEND FOR THE WHOLE PASS**, and it is the shape of the lend that + # keeps D-213's saving. The rows arm needs `rows` off the definition, so it lends the whole + # document; the envelope arm reads no row at all and lends the PROJECTION. Either way the same + # object then serves `object_shares` to the grant legs downstream, so the assembly stops + # reading that bucket once per question asked of it. + if st is None: + st = _ut().lend(session.runtime) if with_rows else _ut().lend_defs(session.runtime) + if with_rows: + # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only + # a READ route passes one — see `scoped_pool`'s own note and contract C5. + pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) + else: + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) + rows_src = [] + + # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it + # as `st`; otherwise this assembly takes its own. Only `user_tables` and `object_shares` are + # served from it, and the assembly writes neither. + ops = _ops(session, table_key, st=st) + # ⭐⭐ W36-T21 — the WRITE half of the field wall. `frozenset()` here meant a column an + # administrator had hidden was still writable through the events transport. + hidden = _ut_hidden(session, table_key, fields_base) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, + admin=session.admin, fallback_ws=None, seen_ids={}, + # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones + # that happen to carry a scoped `table`. + st=session.runtime, + scope_key=table_key, table=ops) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, + consume_corrections=consume_corrections) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, + storage_key=storage_key, fields_base=fields_base) + # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, + # so the closure covers this user's `custom_` and `measure_` columns too. + fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src) + + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "hidden": hidden, + # R9: the Cohorts column's cells from this table's own lists. + "derived": aios_grid.cohort_cells(lists), + "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test + # for is a key a consumer forgets to test for, and this one carries a refusal. + "limits": limits, "defn": defn} + + +def ut_label(defn, key, meta=None): + """THE name of a user table, resolved ONCE (wave 20, item 6a). + + `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that + shows a database name reads through here, because the alternative is what wave 20 found: the + rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker + showed the original (it reads the definition), and neither looked broken. + + ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver + stays because it makes every row already stored — renamed before that fix landed — read + correctly today, without a migration. + """ + return ((meta or {}).get(key, {}).get("name") + or (defn or {}).get("label") or key) + + +def nav_meta(session): + """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" + try: + got = session.runtime.get("nav_meta") + return got if isinstance(got, dict) else {} + except Exception: # noqa: BLE001 + return {} + + +@router.get("/tables") +def list_tables(session: Session = Depends(require_session)): + """This session's user tables — the list the '+ New database' surface renders. + + ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This + route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: + `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on + tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant + with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked + about every table; it is handed the document this function already holds. See + `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not + available. + """ + ut = _ut() + meta = nav_meta(session) + out = [] + tables = ut.all_tables(st=session.runtime) + lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) + for key, t in sorted(tables.items(), + key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): + if not ut.may_open(key, session.uname, session.admin, st=lent): + continue + out.append({"key": key, "label": ut_label(t, key, meta), + "source": t.get("source") or "Blank", + "recordsMutable": ut.records_mutable(key, st=lent), + "createdBy": t.get("createdBy") or "", + "created": t.get("created") or "", + "fields": [dict(f) for f in (t.get("fields") or [])], + "rowCount": len(t.get("rows") or {})}) + return {"tables": out} + + +#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a +#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* +#: +#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since +#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product +#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client +#: could make one, so the whole read-through path was reachable by editing Python — the +#: [[artifact-with-no-importer]] shape, twice burned in this repo already. +#: +#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and +#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that +#: topic can group by; a second list here would be a second definition of the same fact, and the +#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding +#: to a metric KEY instead of carrying SQL. +_ROLLUP_CACHE = {} + + +def _rollup_source_offer(): + """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. + + ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE + time refuses silently — the cells are simply left blank, hours later, on a column that looks + configured. Two exclusions do real work: + * a topic with NO dims cannot be grouped at all, so it can never key a parent row; + * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose + inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: + *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only + ever error. + Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what + the user is matching their own column against, and `payment_state` (a value) and + `partner` (an id) are matched to very different columns. + """ + if _ROLLUP_CACHE.get("offer"): + return _ROLLUP_CACHE["offer"] + from harness import semantic as sem + from harness import windows as W + ut = _ut() + + topics, metrics = sem.topics(), sem.metrics() + by_topic = {} + for key, m in metrics.items(): + # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. + if m.get("agg") in ("ratio", "derived"): + continue + by_topic.setdefault(m["topic"], []).append( + {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", + "description": m.get("description") or ""}) + + out = [] + for tkey, t in sorted(topics.items()): + dims = ((t.get("store") or {}).get("dims") or {}) + measures = by_topic.get(tkey) or [] + if not dims or not measures: + continue + out.append({ + "key": tkey, + "label": t.get("label") or tkey, + "grain": t.get("grain") or "", + "dims": [{"key": dkey, + "label": d.get("label") or dkey, + # `store_query` emits `_id` only when the dim carries a display name + # alongside the key; otherwise the value IS the key. `rollup_sql` handles + # both, and the editor says which so the user matches the right column. + "keyedBy": "id" if d.get("name_col") else "value"} + for dkey, d in dims.items()], + "measures": sorted(measures, key=lambda m: m["label"].lower()), + }) + + # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` + # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds + # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the + # validator refuses would let the editor build a field the save door rejects. + windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} + for k in ut.ROLLUP_SOURCE_WINDOWS] + offer = {"topics": out, "windows": windows} + _ROLLUP_CACHE["offer"] = offer + return offer + + +@router.get("/tables/rollup-sources") +def rollup_sources(session: Session = Depends(require_session)): + """The topic → metric → dim → window offer the rollup field editor renders. + + ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI + matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below + this line it still resolves; added ABOVE it, this endpoint would silently start arriving as + `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — + this is the cheap ordering that keeps it from mattering. + + ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model + files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live + Odoo data" in the field editor and been able to build a column that can only ever be blank, + because there is no mirror behind it. Three comments (here, in `apiBridge` and on the + `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, + and the mode switch's "render only when there is a choice" guard is built on that promise. + ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already + what `refresh` consults to decide whether these tables may exist at all, and a second copy of + the rule would be a second answer the day a tenant gains a mirror. + """ + import odoo_relational + if not odoo_relational.is_royal(session.tenant): + return {"topics": [], "windows": []} + return _rollup_source_offer() + + +@router.post("/tables", status_code=201) +def create_table(body: dict = Body(default=None), + session: Session = Depends(require_session)): + ut = _ut() + body = body or {} + label = str(body.get("label") or "").strip() + if not label: + raise err(400, "bad_label", "give the database a name") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was created") + source = body.get("source") + try: + key = ut.create(label, session.uname, fields=body.get("fields"), + source=source, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was created") + if not key: + raise err(400, "refused", + f"could not create it. The name may be empty or this tenant already has " + f"{ut.MAX_TABLES} databases") + return {"key": key} + + +@router.patch("/tables/{table_key}") +def patch_table(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Rename a database — IN ITS DEFINITION (wave 20, item 6a). + + ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the + nav's display layer; the definition is what the automation editor's database picker, the + schema drawer and every future reader see. A rename that lands in only one of them leaves a + picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an + amendment for whoever owns that door. + """ + defn = _defn_or_refuse(session, table_key) + ut = _ut() + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can rename it") + label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) + if not label: + raise err(400, "bad_label", "give the database a name") + return {"key": table_key, "label": label} + + +@router.get("/tables/{table_key}/footprint") +def table_footprint(table_key: str, session: Session = Depends(require_session)): + """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). + + Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories + without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled + like the delete itself: only someone who could delete may case the joint. + + ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row + COUNT needs the whole document, and only on a materialised table.""" + defn = _defn_or_refuse(session, table_key, defs_only=True) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + s = session.runtime + views, fields = set(), len(defn.get("fields") or []) + try: + bucket = s.get(f"{table_key}_table_workspace") or {} + for _u, ws in bucket.items(): + if isinstance(ws, dict): + views |= set((ws.get("views") or {}).keys()) + fields += len(ws.get("fields") or {}) # per-user custom/measure strata + except Exception: + pass + import core.shares as shares + g = shares.grants("database", table_key, st=s) + auto = [] + try: + import automation_engine as engine + for aid, d in (engine.all_definitions(s) or {}).items(): + if (d.get("config") or {}).get("targetTable") == str(table_key): + auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) + except Exception: + pass + # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where + # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, + # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the + # projection is not what makes it wrong. + # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 + # in a dialog headed "what dies with this database"). Booked rather than fixed here: this + # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` + # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. + if _ut().materialises(table_key, st=s, defn=defn): + rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) + else: + rows = 0 + return {"rows": rows, "fields": fields, "views": len(views), + "sharedUsers": len(g.get("entries") or []), + "automations": sorted(auto, key=lambda a: a["name"].lower())} + + +@router.delete("/tables/{table_key}") +def delete_table(table_key: str, session: Session = Depends(require_session)): + """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). + + ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the + creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role + grantee could reach this route and delete the database somebody shared with them. The wall + is now the definition's own `createdBy`, the same check the rename route always had. + + Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and + DISABLES bound automations with a status note — never deletes them. The client's confirm + dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the + dialog is a product requirement, not a formality.""" + defn = _defn_or_refuse(session, table_key) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + try: + import automation_engine as engine + engine.disable_for_table(session.runtime, table_key) + except Exception: + pass + try: + _ut().delete(table_key, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the delete did not land. Try again") + return {"ok": True} + + +#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. +_IG_FORWARDED = set() + + +def _ig_forward(session): + """Bring this tenant's Instagram tables onto the current schema, at most once per process. + + ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE + path. `ut_ensure` calling it is right for a schema an automation is about to append to, and + useless for a change a PERSON is waiting to see: the owner's report was *"the first field is + still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. + The write path stays exactly as it was — this is a second door to the same idempotent call, + not a replacement for it. + + ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per + tenant per process; `migrate_ig_tables` returns without a write when every table is already + current (the common case after the first read); and a failure is SWALLOWED — a migration must + never be the reason a database will not open. + + ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be + retried on every subsequent read of every table for the life of the process — the write path is + still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing + store call on the hot path of a grid that is trying to render. + """ + tenant = str(getattr(session, "tenant", "") or "") + if tenant in _IG_FORWARDED: + return + _IG_FORWARDED.add(tenant) + try: + import automation_engine as engine + engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") + + + +#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized +#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is +#: not — the shape this exists for is one already-paid provider payload per row. +JSON_LIST_MAX = 400 + + +def _thin_json(fields, merged, table_key=""): + """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. + + ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads + `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something + large rather than something empty. `_truncated` is what the viewer keys its fetch on. + + ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the + viewer needs no table key, no record id and no new props threaded down through three + components to find the document — the route that removed the value says where it went. One + writer of that address instead of a server rule and a client rule that must agree forever. + """ + json_keys = [str(f.get("key")) for f in (fields or []) + if str(f.get("type") or "") == "json"] + if not json_keys: + return merged + out = {} + for pid, cells in (merged or {}).items(): + row = cells + for key in json_keys: + raw = cells.get(key) + if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: + if row is cells: + row = dict(cells) + row[key] = json.dumps({ + "_truncated": True, "bytes": len(raw), + "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) + out[pid] = row + return out + + +@router.get("/tables/{table_key}/rows") +def table_rows(table_key: str, session: Session = Depends(require_session)): + """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, + identity}` — so the client's generic topic fetch consumes it with zero new parsing. + + ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell + from the OVERLAY stratum only (that is what makes a custom column render standalone) — a + user table's base values live in its DEFINITION rows, so they are layered UNDER the user's + overlay edits here: base first, overlay wins. Without this every base cell reads empty + (found by this route's own gate check, not by luck).""" + import aios_grid + + _ig_forward(session) + # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. + # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE + # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), + # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask + # about the SAME document in the SAME request. The lend is the fix W31 QA already built for + # `_defn_or_refuse`; this threads it through the three sites that never got it. + # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a + # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the + # value it replaced (contract C5). + _lent = _ut().lend(session.runtime) + g = ut_assembly(session, table_key, st=_lent) + # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge + # this line does not perform since the route was written (owner item 3, 2026-08-09: + # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). + # + # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` + # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table + # is overlay-typed by construction, so it looked up a key that could not be there and every + # such cell rendered blank. + # + # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds + # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed + # in the store exactly as they should. Only the READ-BACK dropped them, which is why the + # value survived the gesture, vanished on reload, and looked like "it reset itself" — and + # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. + # + # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition + # value must not shadow an edit the user has made on top of it. + # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this + # widens what a caller can SEE by exactly their own edits and nothing else. + _overlays = (g.get("ws") or {}).get("overlays") or {} + merged = {} + for _r in g["rows_src"]: + _pid = str(_r["pid"]) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _ov = _overlays.get(_pid) + if isinstance(_ov, dict): + _cells.update(_ov) + merged[_pid] = _cells + # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE + # JSON DOCUMENTS DO NOT RIDE THE LIST. + # + # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% + # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB + # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The + # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole + # vendor response crossed the wire, was parsed by the browser and held in memory purely so a + # clipped first line could be drawn. + # + # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by + # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — + # the one place a person actually reads it, for the one row they opened. The cell that rides + # the list is a VALID small document saying what it stands for, so the preview renders + # honestly instead of showing half a truncated brace. + # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, + # so a tenant using `json` for a short config sees no change at all. + rows = aios_grid.rows_from_pool( + g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) + # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ + # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. + _stamp_ai_states(table_key, g["fields"], rows, st=_lent) + # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you + # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this + # database says so here, with its cause and the recommendation, rather than waiting to be + # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the + # honest answer for a table nothing limits — never an absent key, which a client cannot tell + # apart from an older server. + _report = _ut().limit_report(table_key, st=_lent) + # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the + # only thing that ever set this key; the write door never stopped working and every client + # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a + # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the + # customer scope — `grid_events.docs_for` — never a twin here. + from core import grid_events as _ge + return {"fields": g["fields"], "rows": rows, "today": g["today"], + "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, + admin=session.admin, st=session.runtime), + "pulled_at": time.strftime("%Y-%m-%d %H:%M"), + "identity": {"pid": "pid"}, + "scope": {"table": table_key, "rowCount": len(rows)}, + "limits": [_report] if _report else [], + "recordsMutable": _ut().records_mutable(table_key, st=_lent)} + + +#: The per-cell provenance a row carries on the wire, one key per enrichment column. +#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. +#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new +#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces +#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this +#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the +#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off +#: every field key, so no column can ever be called `_ai_*`. +AI_STATE_PREFIX = "_ai_" + + +def _stamp_ai_states(table_key, fields, rows, st=None): + """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. + + ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token + count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest + would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary + is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the + RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. + + ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no + enrichment column is untouched, and a freshly created column adds nothing until something + runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row + instead of against whatever was true when the value was written. + """ + cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] + if not cols: + return rows + import ai_enrich as _ae + for field in cols: + col = str(field.get("key") or "") + marks = _ut().ai_enrich_marks(table_key, col, st=st) + cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} + for row in (rows or []): + if not isinstance(row, dict): + continue + state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) + if state != "empty": + row[AI_STATE_PREFIX + col] = state + return rows + + +@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") +def table_cell(table_key: str, pid: str, fkey: str, + session: Session = Depends(require_session)): + """ONE cell, whole — the other half of `_thin_json`. + + ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor + for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON + viewer opens this for the one row a person is actually reading. + + ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's + view of the table), so this cannot become a side door onto a table the caller may not open — + which is the failure a "just fetch the raw cell" helper invites. + ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must + read back what they typed, not the definition value underneath it. + """ + g = ut_assembly(session, table_key) + field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) + if field is None: + raise err(404, "unknown field", f"{fkey!r} is not a column on this database") + row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) + if row is None: + raise err(404, "unknown record", f"no record {pid!r} in this database") + overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} + value = overlay.get(fkey, row.get(fkey)) + return {"table": table_key, "pid": str(pid), "field": str(fkey), + "value": "" if value is None else str(value)} + + +@router.post("/tables/{table_key}/rows", status_code=201) +def add_row(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). + + ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo + that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the + response rather than assume; the client re-anchors on what came back. + """ + _records_or_refuse(session, table_key) + ut = _ut() + values = (body or {}).get("values") or {} + if not isinstance(values, dict): + raise err(400, "bad_values", "values must be an object of {fieldKey: value}") + try: + rid = ut.add_row(table_key, values, session.uname, st=session.runtime, + rid=(body or {}).get("rid")) + except Exception: + raise err(503, "store_unavailable", "the row was not saved. The store refused") + if rid is None: + # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap + # sentence alone would misdirect — the reader would go and count rows. Ask the same + # validator the law used rather than re-deciding here (one rule, two voices). + pf = ut.profile_field(table_key, st=session.runtime) + if pf and pf["key"] in values: + _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) + if not ok: + raise err(400, "refused", + f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " + f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " + f"profile link (instagram.com/name)") + raise err(400, "refused", + f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") + _refresh_relations(session) + return {"rid": rid, "pid": int(rid)} + + +@router.post("/tables/{table_key}/rows/import", status_code=201) +def import_rows(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. + + Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a + spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, + nothing is matched or overwritten, and the dialog says so before the button is pressed. + + ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate + the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in + an import is not a stray to be tidied away — it means the client offered a target it should + not have, and silently dropping it would leave the user looking for a column of values that + never arrived. The refusal names the column. + + ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every + profile cell validates, because a half-imported file is the worst outcome available: the user + cannot tell which rows landed without reconciling the spreadsheet by hand. + """ + _records_or_refuse(session, table_key) + ut = _ut() + rows_in = (body or {}).get("rows") + if not isinstance(rows_in, list) or not rows_in: + raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") + if any(not isinstance(r, dict) for r in rows_in): + raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") + defn = _defn_or_refuse(session, table_key) + by_key = {f["key"]: f for f in (defn.get("fields") or [])} + asked = {k for r in rows_in for k in r} + unknown = sorted(k for k in asked if k not in by_key) + if unknown: + raise err(400, "unknown_field", + f"this database has no column {unknown[0]!r}") + computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) + if computed: + label = by_key[computed[0]].get("label") or computed[0] + raise err(400, "computed_field", + f"{label!r} is worked out from other columns, so it cannot be imported into") + # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. + # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a + # second client, or a future importer met no wall at all and the string landed verbatim in a + # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE + # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every + # rule on this route exists to prevent. The sentence names the row and the column, because + # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. + for index, row in enumerate(rows_in): + for key, value in row.items(): + why = ut.cell_type_refusal(by_key[key], value) + if why: + raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") + try: + made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "nothing was imported. The store refused") + if made is None: + raise err(400, "refused", + f"nothing was imported. {len(rows_in)} rows would take this database past " + f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") + _refresh_relations(session) + return {"imported": len(made), "pids": [int(r) for r in made]} + + +# --------------------------------------------------------------------------------------------- +# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) +# --------------------------------------------------------------------------------------------- +# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same +# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's +# definition to everyone without handing over the table. This supersedes wave 17's "fields are +# per-user" law for this path only; the connector scopes keep their own model. +# +# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the +# per-user workspace overlay, which is why the automation editor's "Automation column" picker +# could not see a column the user had just created — it reads the DEFINITION. Same defect shape +# as the rename (item 6a): two places to look, and the surfaces disagreed silently. + +def _field_or_refuse(session, table_key, fkey=""): + """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. + + ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and + `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they + write through the `user_tables` write doors), so no projected snapshot survives into a write. + ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 + draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" + defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + ut = _ut() + if not ut.is_user_table(table_key, st=session.runtime): + raise err(400, "not_a_user_table", + "only a user-created database has an editable schema. A connected source " + "owns its own columns") + # ⭐⭐ W36-T21 — A HIDDEN COLUMN IS NOT EDITABLE, AND THIS IS THE DOOR THAT HAD TO SAY SO. + # `EventCtx.hidden_keys` walls the events transport; the REST schema routes (rename, retype, + # delete a column) do not pass through it at all. Without this an account that could not SEE + # `unit_cost` could still DELETE it for the whole tenant — the loudest possible version of a + # wall that exists on one wire only. ⚠ Read the closure off the DECLARED fields, which is what + # `routes_admin` validates a stored `hiddenFields` list against. + if fkey and str(fkey) in _ut_hidden(session, table_key, defn.get("fields") or []): + raise err(403, "forbidden", + "this database has no column by that name that you may edit") + if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, + st=session.runtime): + field = next((f for f in (defn.get("fields") or []) + if f.get("key") == str(fkey)), None) + if isinstance((field or {}).get("automation"), dict) \ + and field["automation"].get("preset") is True: + # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not + # ut.may_edit_field` above is the wall). Said out loud because the sentence itself + # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a + # blanket "pre-set fields are locked" would now be the server explaining a refusal + # it did not make — `preset_editable` is the one predicate that answers this. + raise err(403, "preset_field_locked", + "this is a pre-set column, so its name and type are fixed; you may sort, " + "filter or hide it, edit any Rollup column, and add your own columns") + raise err(403, "forbidden", "that column can only be changed by the database's creator " + "or an admin") + if not fkey and not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can add a column") + return defn + + +@router.post("/tables/{table_key}/fields", status_code=201) +def add_field(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _field_or_refuse(session, table_key) + ut = _ut() + field = ut.add_field(table_key, body or {}, st=session.runtime) + if not field: + # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers + # None for every refusal, so this route said "check the name and type" to somebody whose + # name and type were fine and whose automation column named a flow that does not exist. + # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the + # one thing that was never wrong. Checked HERE, in the route's own words, because the + # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never + # re-implements it (a second copy of the rule is how two doors start disagreeing). + raise err(400, "refused", + _refusal_sentence(ut, session, body or {}, table_key=table_key)) + if field.get("type") == "link": + synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) + field = synced.get("field") or field + # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route + # became reachable for one. It was gated on `link` alone, while `patch_field` and + # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first + # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). + # The user would have created the column, watched a 201 come back, and read a blank cell for + # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. + # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in + # the tenant, and adding a text column has nothing to fold. The condition is now "is this + # field relational", which is the question that was always meant. + if field.get("type") in ("link", "rollup"): + _refresh_relations(session) + return _with_dropped(ut, {"field": field}, body) + + +def _with_dropped(ut, out, body): + """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). + + ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag + cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and + `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the + defect. T51's contract is that an unknown config key is dropped **and named**, so the naming + rides the response beside the accepted field rather than inside the validator. + + ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to + ignore the key, which is how a report stops being read before it stops being true. + """ + dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) + if dropped: + out = dict(out) + out["dropped"] = dropped + return out + + +def _refusal_sentence(ut, session, body, table_key="", fkey=""): + """Why was this column refused? The specific reason when we can name one, the general list + otherwise — never a specific-sounding guess.""" + # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag + # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment + # column would otherwise be explained by the flow law -- "pick a flow, or make this an + # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user + # never touched. + if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": + bag = (body or {}).get("aiEnrich") + if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): + return ("an AI enrichment column needs a prompt. It is the only thing that can " + "produce a value here, so a column without one would stay empty forever") + # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is + # what the contract asks for and what makes it actionable — "at most one" sends the reader + # hunting through a 40-column schema for a flag they cannot see from the header. + if isinstance((body or {}).get("profile"), dict): + # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at + # once (an `int` profile column on a table that already has a profile column), and the + # TYPE is the one that is wrong about what the caller just sent — unconditionally, no + # matter what else is on the table. Answering "you already have one" to somebody whose + # real mistake was the column type sends them to fix the wrong thing, which is the + # misdirection D-46 closed one door over. + if str((body or {}).get("type") or "text").strip().lower() != "text": + return ("a profile column is a flag on an ordinary TEXT column. It validates what " + "is typed into it, which it can only do for text") + existing = ut.profile_field(table_key, st=session.runtime) if table_key else None + if existing and existing.get("key") != str(fkey): + return (f"this database already has a profile column: " + f"{existing.get('label') or existing.get('key')!r}. A database has at most " + f"one, so the automation knows which handle to enrich; edit that column, or " + f"take the flag off it first") + bag = (body or {}).get("automation") + if isinstance(bag, dict): + flow = str(bag.get("flowId") or "").strip() + if not flow: + return ("an automation column has to name the automation that fills it. Pick a " + "flow, or make this an ordinary column") + if not ut.flow_bound(bag, st=session.runtime): + return (f"this column names automation {flow!r}, which does not exist in this " + f"workspace. It may have been deleted; pick a flow that is still there") + kind = str((body or {}).get("type") or "").strip() + if kind and kind not in ut.UT_FIELD_TYPES: + return (f"{kind!r} is not a column type here (types: " + f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") + return (f"the column was refused. Check the name and type, or the table may be at its " + f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") + + +@router.patch("/tables/{table_key}/fields/{fkey}") +def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Edit one column's definition, and MIGRATE its values when options are renamed. + + ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: + [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed + Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and + every saved view that filtered on it. + """ + _field_or_refuse(session, table_key, fkey) + ut = _ut() + body = body or {} + migrated = None + renames = body.get("renames") + if renames: + try: + migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the rename did not land. Try again") + # The per-user workspace strata and any view filter naming the old value are the OTHER + # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an + # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a + # missing counterpart must not lose the half that DID land. + try: + import core.table_store as table_store + fn = getattr(table_store, "rename_choice_values", None) + if callable(fn): + fn(table_key, fkey, renames, st=session.runtime) + migrated = dict(migrated or {}, workspace=True) + except Exception: # noqa: BLE001 + migrated = dict(migrated or {}, workspace=False) + field = ut.patch_field(table_key, fkey, body, st=session.runtime) + if not field: + # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as + # adding one, so it must get the same sentence naming the column that already holds the + # flag — a patch that answered "check the name and type" would send the reader to the + # one thing that was never wrong (the D-46 lesson, one door over). + raise err(400, "refused", + _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) + synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) + field = synced.get("field") or field + _refresh_relations(session) + out = {"field": field} + if migrated is not None: + out["migrated"] = migrated + return _with_dropped(ut, out, body) + + +def _fire_on_change(table_key, pid, changed, session): + """Run any `on_change` enrichment column whose prompt names a cell that just moved. + + ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. + `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a + 20,000-row import into 20,000 whole-document reads on the single process this product runs + (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and + takes the definition, so this reads once and asks about every column. + + ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; + an enrichment that could not run is a missing value, not a lost edit, and the run's own report + carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a + fan-out here would put a vendor call on the critical path of every keystroke-commit. + """ + import ai_enrich as _ae + + try: + defn = _ut().get(table_key, st=session.runtime) or {} + wanted = _ae.on_change_fields(defn, changed.keys()) + for field in wanted: + # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still + # somebody's edit spending somebody's tokens, so it is booked against the person who + # typed rather than left unattributed. + _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], + user=session.uname) + except Exception: # noqa: BLE001 + pass + + +@router.post("/tables/{table_key}/fields/{fkey}/enrich") +def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. + + ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides + (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the + tenant by opening a grid. + + `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the + value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic + plan, where the policy and the never-overwrite-a-human law both apply. The two are one + function with one flag, not two runners. + + ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by + reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a + ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a + populated `skipped` is a correct, informative answer, and the client must render it rather + than treat it as success. + """ + _field_or_refuse(session, table_key, fkey) + import ai_enrich as _ae + + rows = (body or {}).get("rows") + if rows is not None and not isinstance(rows, list): + raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " + "rows this column's own settings choose") + # The caller's permitted pool, the same one the row doors use. A named row outside it is + # dropped rather than refused: a stale client naming a record that has been deleted or moved + # out of scope should not fail a run over the rows it can legitimately fill. + if rows is not None: + allowed = {str(p) for p in scoped_pids(session, table_key)[0]} + rows = [str(r) for r in rows if str(r) in allowed] + # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" + # sends `always`. Anything else falls back to the column's own saved policy rather than to a + # default, so a typo cannot quietly widen what a run touches. + report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, + policy=str((body or {}).get("scope") or "") or None, + # A named row set through THIS door is a person asking. + manual=rows is not None, + # ⭐ W35-T41 / C7 — the usage ledger's attribution. + user=session.uname) + if report.get("problem"): + # A run that could not start at all is not a 200: nothing was attempted, nothing was + # spent, and the reason is actionable (no provider configured, or the wrong column). + raise err(400, "enrich_refused", str(report["problem"])) + return report + + +@router.delete("/tables/{table_key}/fields/{fkey}") +def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): + _field_or_refuse(session, table_key, fkey) + if not _ut().delete_field(table_key, fkey, st=session.runtime): + raise err(400, "refused", + "that column could not be removed. A database must keep at least one") + _refresh_relations(session) + return {"deleted": fkey} + + +@router.delete("/tables/{table_key}/rows/{rid}") +def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): + # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. + # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* + # A single DELETE was FIVE whole-document deep copies before the commit even began — three in + # the guard (`get` + `may_open` + `records_mutable`) and two more inside + # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads + # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** + # (D-185), so the guard alone was seconds of copying to answer questions about one row. + # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` + # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document + # under the store lock, so a lent snapshot can never be the thing written back. + # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually + # consistent delete is indistinguishable from one that did not work. This makes the guard + # cheap; it does not make the commit optimistic. + lent = _ut().lend(session.runtime) + _records_or_refuse(session, table_key, st=lent) + # ⭐⭐ W36-T21 — THE ROW SCOPE, ON THE DELETE DOOR. `patch_row` has asked this since it was + # written (`pid not in g["pids"]` -> 403) and this door never did, because until now every + # account that could open a `ut_*` database could see every row of it. The moment an + # administrator can row-scope one, "may not SEE row 5" and "may DELETE row 5" become two + # different answers unless this is here — and delete is the one that cannot be undone. + # ⚠ Guarded on `row_scope_applies` so an unscoped account pays nothing: for them the pid set + # is every row and the question has one answer. + import core.perm_scope as _ps + if _ps.row_scope_applies(session.user, table_key): + _pids, _f, _d = scoped_pids(session, table_key) + if not str(rid).isdigit() or int(rid) not in _pids: + raise err(403, "out_of_scope", "that row is not in this database") + try: + ok = _ut().delete_row(table_key, rid, st=lent) + except Exception: + raise err(503, "store_unavailable", "the delete did not land. Try again") + if not ok: + raise err(400, "refused", "rows can only be deleted from user-created databases") + _refresh_relations(session) + return {"ok": True} + + +@router.patch("/tables/{table_key}/rows/{pid}") +def patch_row(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Cell edits — the products PATCH on the user-table ctx. Routed through + `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; + the accepted values are read BACK from the bucket, never echoed from the request.""" + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + _records_or_refuse(session, table_key) + g = ut_assembly(session, table_key, consume_corrections=False) + if pid not in g["pids"]: + raise err(403, "out_of_scope", "that row is not in this database") + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, + # ⭐⭐ W36-T21 — the assembly's OWN closure, not an empty set. `g["fields"]` is already + # stripped, but `grid_events` asks `ctx.hidden_keys` by name: a PATCH naming a hidden + # column would otherwise be accepted on a payload that never showed it. + hidden_keys=g.get("hidden") or frozenset(), + st=session.runtime, # R6b (D-16) + scope_key=table_key, table=_ops(session, table_key)) + try: + grid_events.handle_one( + {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable. Your change was not saved") + # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. + # + # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH + # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. + # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's + # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began + # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` + # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring + # claiming otherwise is what makes the next reader look for a merge bug that is not here. + # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY + # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` + # asks whether THIS write landed, and this write goes to the definition. + stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) + .get(str(pid)) or {}) + accepted = {k: stored.get(k) for k in updates if k in stored} + + def _took(k): + """Did the cell TAKE this write? Normally that is "stored == asked". + + ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` + and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell + the client to roll back a write that landed. But it cannot simply be exempted either — + a junk handle leaves the OLD value sitting in `stored`, which would then read as + accepted. So the question asked is the exact one: **is what is stored the canonical form + of what was asked?** Anything else is a genuine refusal. + """ + if k not in accepted: + return False + want = str(updates[k]) + if stored.get(k) == want: + return True + pf = _ut().profile_field(table_key, st=session.runtime) + if pf and pf["key"] == k: + handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) + return bool(ok) and stored.get(k) == handle + return False + + refused = sorted(k for k in updates if not _took(k)) + # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write + # this cell?" is not recoverable from the value afterwards, so the only place to record it is + # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic + # run overwrite it, whatever the column's `overwrite` policy says. + # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused + # write `human` would freeze a cell against the agent on the strength of an edit that never + # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. + took = {k: v for k, v in accepted.items() if k not in refused} + if took: + try: + _ut().note_human_edit(table_key, took, pid, st=session.runtime) + except Exception: # noqa: BLE001 + # Provenance is metadata about a write that has already succeeded. Failing the + # request here would tell the user their edit was lost when it was not. + pass + _fire_on_change(table_key, pid, took, session) + out = {"pid": pid, "updates": accepted} + if refused: + out["refused"] = refused + # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a + # profile blank cleared. Without this the grid keeps painting a stale follower count under + # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other + # half: the client may keep only what the server actually took, and must be TOLD what else + # moved. Derived by diffing this row against what was asked for, so it cannot drift from + # whatever the clear rule decides to touch. + also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} + cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) + if cleared: + out["cleared"] = cleared + # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, + # amendment A5; SESSION B built and gated the function and correctly declared it INERT until + # this line existed, citing [[flag-shipped-without-its-writer]]). + # + # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead + # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the + # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we + # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try + # it again, the account is back". Without this call that person has no way back at all, and + # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. + # + # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this + # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a + # human touched the handle cell. + # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- + # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must + # be readable in the log rather than degrade into "the re-arm quietly stopped working". + _pf = _ut().profile_field(table_key, st=session.runtime) + if _pf and _pf["key"] in updates: + try: + import automation_engine as _engine + _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") + _refresh_relations(session) + return out + + +# ── CONTRACT C1 (W36-T20): THE MIRROR READER ────────────────────────────────────────────────── +# ⭐⭐ R6. `core.perm_scope.scoped_table` is the ONE door to any database's rows and it answers for +# a MATERIALISED `ut_*` table entirely on its own — deliberately, so a cold process (E's sandbox +# subprocess, a worker, a gate) that never imported a route still gets the right answer. A +# READ-THROUGH grid is the one arm it cannot serve alone: those ten databases store no rows in the +# tenant document at all, and their rows live in the DuckDB mirror behind `routes_odoo_tables`, +# two layers above `core`. +# +# ⛔ ONE FETCH, NOT A SECOND ONE. This hands C1 the SAME `_read_through_rows` that `scoped_pool` +# and `scoped_pids` already share, so the rows a script sees through C1 are byte-for-byte the rows +# the grid sees — by construction, not by a second query that agrees today (W31-T20's argument, one +# caller further out). +# +# ⚠ AND `TooBigToMaterialise` BECOMES A REPORTED REFUSAL, NEVER A SHORT ANSWER. Standing rule 1's +# second sentence, and the shape is `_PID_SCOPE_LIMIT`'s so nothing downstream needs a second +# vocabulary for it. +def _c1_mirror_rows(table_key, field_keys, st): + """C1's mirror arm: `perm_scope.register_mirror`'s reader over `_read_through_rows`.""" + import core.perm_scope as perm_scope + + try: + return _read_through_rows(table_key, field_keys, rt=st) + except _too_big() as e: + raise perm_scope.Unresolvable( + cause=str(e), + recommendation=_PID_SCOPE_LIMIT["recommendation"], + subject="rows", effect="unresolved") from e + except RuntimeError as e: + raise perm_scope.Unresolvable( + subject="rows", effect="unreadable", cause=str(e), + recommendation="the connector mirror is not ready on this process; retry once the " + "store has finished opening") from e + + +def _register_mirror(): + """Declare the mirror reader to C1. Called at import; returns True once it is registered.""" + import core.perm_scope as perm_scope + return perm_scope.register_mirror(_c1_mirror_rows) + + +_C1_MIRROR = _register_mirror() diff --git a/api/routes_web_agent.py b/api/routes_web_agent.py index b9d119709bbbcf2501c6b372f0165fb5af1ea6c1..57453ded998810e7046aa5d14acae4d739fd235b 100644 --- a/api/routes_web_agent.py +++ b/api/routes_web_agent.py @@ -22,10 +22,11 @@ admin-gated, because it spends money and because D-51 §5's authorisation postur the tenant is authorised to use, at their instruction") is not something an ordinary member should be able to commit the tenant to. -⛔ THIS ROUTER IS NOT MOUNTED YET. `main.py` belongs to another lane this wave, so the one -`app.include_router(routes_web_agent.router)` line is a cross-fence ask — and -`verify_web_agent.py` FAILS until it lands, deliberately: three finished routers once shipped -404-dead behind entirely green gates, and a gate that tolerates it is how that happens twice. +⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and +`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting +either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which +is the same class of stale claim as a green gate on a router nobody wired: three finished routers +once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. """ from fastapi import APIRouter, Body, Depends diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 8fb2db4f0aa9d0331ead406ab80a14bf146011f7..80b23e386a224df3567eb29d551b7eaa22c9f2cd 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -1,2256 +1,2256 @@ -"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. - -This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the -standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit -component. The React bundle reads its data from `window.__AIOS_GRID__` (an object -`{fields, rows}`) that we inject into the page before the app's module script runs -— so there is NO backend and NO /api call in the Streamlit container; the browser only ever -sees the derived JSON we hand it. - -Usage (from any page): - import aios_grid - aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) - -Design notes: - * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and - `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and - sidesteps deploy_hf.py's import guard entirely. - * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to - deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy - dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py). - For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. - * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args - into the React app and returns guarded events for persistence in the tenant store. The old - injected-HTML path remains a read/local-write fallback when component assets are absent. -""" -import json -import re -from pathlib import Path - -_HERE = Path(__file__).resolve().parent - -# Where the inlined single-file build lives. FIRST match wins: -# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) -# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) -_EMBED_CANDIDATES = [ - _HERE / "aios_grid_embed.html", - _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", -] -_COMPONENT_CANDIDATES = [ - _HERE / "aios_grid_component", - _HERE.parent / "aios-web" / "web" / "dist-embed", -] -_DECLARED_COMPONENTS = {} - -# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS -# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is -# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. -# -# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so -# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the -# module preserves that exactly — no import, no guard interaction — while still single-sourcing -# the values (a build-time copy from another tree would reintroduce the drift we're removing). -# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run -# aios-web/verify_fields_contract.py. --- -_FIELDS_PATH = _HERE / "aios_grid_fields.json" - - -def _load_fields(): - if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " - "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." - ) - doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) - fields = doc.get("fields") if isinstance(doc, dict) else doc - if not isinstance(fields, list) or not fields: - raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") - return fields - - -FIELDS = _load_fields() - - -def product_fields(): - """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same - JSON). A separate accessor rather than a second module constant so the one file-read and the - one failure mode stay shared with `FIELDS`.""" - doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) - fields = (doc.get("product_data") or {}).get("fields") - if not isinstance(fields, list) or not fields: - raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.") - return fields - -# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — -# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. - - -def _round(v): - return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v - - -# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors -# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. -# select — a single-select with its own `options` (the "Status" a user wants to add; distinct -# from the Odoo-sourced `status` lifecycle, which is not user-defined) -# user — an assignee, whose choices come from the HOST's real user list, never from here -#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared -#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every -#: member of it (the `multi` grouping/cell contract the Cohorts column established). -#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · -#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level -#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's -#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). -#: ⭐ Wave-19 R7 / contract C5: `image` — a PICTURE on a record. The cell holds a string -#: REFERENCE, never bytes: a product `code`, an `ed:` editorial asset, or `rec:` for -#: something uploaded through the field (`POST /api/v1/assets/records`). Storing a reference is -#: what lets Royal's 1,142 existing SKU masters appear with nothing re-uploaded, and it keeps the -#: overlay stratum the size it is — base64 bytes in a JSON blob read on every render would be a -#: megabyte-per-row tax on the whole store. Editable by NATURE (deliberately NOT in -#: `READONLY_CUSTOM_TYPES`): the ref is what the upload endpoint hands back, and the client PATCHes -#: it through the ordinary overlay wall rather than the asset route writing cells behind it. -#: ⭐ WAVE 23 (C7) — `json` joined: a cell holding a whole DOCUMENT (an Instagram comment thread, -#: a webhook payload, a scraped blob) that opens in its own viewer instead of being flattened -#: into one unreadable line. It is EDITABLE by nature, like `image`: the value is still a plain -#: string on the wire, so it rides the ordinary overlay wall — what makes it a json field is that -#: `grid_events` REFUSES a write that does not parse (a column promising structure must not -#: silently hold something that isn't). -#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They ride here because -#: `UT_FIELD_TYPES` must stay a SUBSET of this set (gated in verify_api's W18-UT section) — the -#: surfaces that CREATE them are the user-table databases, where a relation between two tables is -#: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to -#: point at, so the column menu there simply never offers one. -#: ⭐⭐ WAVE-34 (owner ruling R13) — `ai_enrich` JOINS, and it had to join HERE in the same change -#: that put it in `UT_FIELD_TYPES`, not a ticket later. The wave planned a SERVER-FIRST landing on -#: the reasoning that a kind the client does not offer is invisible while a kind the server refuses -#: deletes a column. That reasoning is sound and the conclusion was still wrong, because THREE -#: parity gates chain over these sets and none of them permits a partial landing: -#: `verify_api` W18-UT `UT_FIELD_TYPES - CUSTOM_FIELD_TYPES == set()` (this line) -#: `aw_fields_contract` §5 `CREATABLE_TYPES == CUSTOM_FIELD_TYPES`, EXACT set equality -#: `types.ts` `CREATABLE_TYPES: readonly FieldType[]`, so the union must carry it too -#: Measured live by lane B at 17:09: `api_api` went 969/969 to 968/969 the moment the kind entered -#: `core/user_tables.py` alone, printing `got {'ai_enrich'} want set()`. The comments on -#: `json`, `link`/`rollup` and `code` below all say the same thing in their own words. -CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", - "checkbox", "phone", "email", "url", "rating", "created_time", "formula", - "automation", "image", "json", "link", "rollup", "code", "ai_enrich"} - -#: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. -#: -#: R13 is explicit that this kind is "syntax-highlighted storage + language config, NO execution -#: engine". So a language is a RENDERING hint and nothing else: it selects a highlighter, it never -#: selects an interpreter, and no value here may ever grow a run path. The list is short on -#: purpose — every entry costs a highlighter the client actually has to implement, and an -#: unimplemented language would paint plain text under a label promising colour. -#: -#: `plain` is the default and the fallback, so it is never a second way to say nothing: it is the -#: honest answer for a snippet whose language the user has not chosen. -CODE_LANGUAGES = {"plain", "json", "sql", "python", "javascript", "typescript", - "html", "css", "markdown", "yaml", "xml", "shell"} - - -def _clean_code(raw): - """The `code` field's config bag -> `{'language': ...}`, or None. - - Deliberately OPTIONAL rather than required (the `automation` posture, not `link`'s): a code - column with no declared language is a legitimate state — it stores and highlights as plain - text — so refusing the FIELD over a missing bag would block the ordinary create path. An - unknown language falls back to `plain` rather than refusing, because the value is a rendering - hint: dropping the user's column to punish a typo in a highlighter name would be the - disproportionate half of the fail-closed rule. - - ⚠ `plain` RETURNS NONE, and that is what makes the control reversible rather than one-way. - Absent already means plain, so storing `{'language': 'plain'}` would be the default wearing a - second name (the `kanbanClamp` law). But the patch path resolves an OMITTED key to the - previous value — so if plain were merely omitted by the client, switching a column back from - SQL to Plain text would keep storing SQL and read as a control that does not save. Sending - the bag explicitly and having it evaluate to None here means: omit = keep, plain = clear. - """ - if not isinstance(raw, dict): - return None - lang = str(raw.get('language') or 'plain').strip().lower() - if lang not in CODE_LANGUAGES or lang == 'plain': - return None - return {'language': lang} -#: User-created types whose CELLS are read-only: their values are computed (formula — client -#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). -#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so -#: the client never offers an editor and the host never accepts a cell write for them. -READONLY_CUSTOM_TYPES = {"created_time", "formula"} -MAX_FIELD_OPTIONS = 50 -MAX_FORMULA_LEN = 500 -#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. -RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 - -#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that -#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's -#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata -#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a -#: measure field is READ-ONLY and its values are computed by the host per render. -MEASURE_FIELD_PREFIX = "measure_" -#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). -MEASURE_FIELD_TYPES = {"currency", "int", "pct"} - - -def _clean_options(raw): - """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. - Mirrors types.ts parseOptions — a choice list that means one thing in the picker and - another in the filter dropdown is a column with two vocabularies.""" - out, seen = [], set() - for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: - if not isinstance(v, (str, int, float)) or isinstance(v, bool): - continue - s = str(v).strip()[:120] - if not s or s.lower() in seen: - continue - seen.add(s.lower()) - out.append(s) - return out[:MAX_FIELD_OPTIONS] - - -def _clean_option_colors(raw, options): - """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary.""" - if not isinstance(raw, dict): - return {} - supplied = {} - for label, color in raw.items(): - if not isinstance(label, str) or not isinstance(color, str): - continue - clean = color.strip().upper() - if re.fullmatch(r"#[0-9A-F]{6}", clean): - supplied[label.strip().lower()] = clean - out = {} - for option in options or []: - color = supplied.get(str(option).strip().lower()) - if color: - out[str(option)] = color - return out - - -def _choice_appearance(raw, options): - """Validated select-family appearance. Absent colour toggle means legacy-on.""" - if not isinstance(raw, dict): - return {} - out = {} - if isinstance(raw.get("colorCodeOptions"), bool): - out["colorCodeOptions"] = raw["colorCodeOptions"] - colors = _clean_option_colors(raw.get("optionColors"), options) - if colors: - out["optionColors"] = colors - return out - - -def _clean_rating_max(raw): - """A rating's star count, bounded. Anything unusable is the default, not a refusal — the - field still holds its 1..max integers either way.""" - try: - return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) - except (TypeError, ValueError): - return RATING_MAX_DEFAULT - - -#: The field types a number-style display format may apply to. `formula` is here because its -#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. -_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} - - -def _clean_format(raw, ftype): - """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong - types return None (the property is simply absent). Rendering-only — a format can change how - a value reads, never what it is, which is why this needs no parity gate of its own.""" - if not isinstance(raw, dict): - return None - out = {} - if ftype in _NUMBER_FORMAT_TYPES: - if isinstance(raw.get("thousands"), bool): - out["thousands"] = raw["thousands"] - if raw.get("decimals") is not None: - try: - d = int(raw["decimals"]) - except (TypeError, ValueError): - d = None - if d is not None and 0 <= d <= 4: - out["decimals"] = d - if isinstance(raw.get("abbrev"), bool): - out["abbrev"] = raw["abbrev"] - elif ftype in ("date", "created_time"): - if isinstance(raw.get("time"), bool): - out["time"] = raw["time"] - if raw.get("tz") in ("local", "utc"): - out["tz"] = raw["tz"] - return out or None - - -def _clean_permissions(raw): - """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is - the host handler's business (creator/admin, enforced fail-closed there); this validates - only the shape, like every other property here.""" - if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): - return {"edit": raw["edit"]} - return None - - -#: ⛔ THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is -#: refused by a wall that is supposed to be structural (2026-08-03). -#: -#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client -#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` — CONCATENATE, TEXT, -#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so -#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace` -#: dropped the column on read. Nothing went red — a refused create looks like a quiet failure -#: and a dropped column looks like a column nobody made. -#: -#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now", -#: "OK")` and could not be created through the product UI at all. -#: -#: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar — that is the filter_sql-class -#: drift risk this file's docstring names. The charset says which characters may appear; the -#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote -#: character brings with it. -_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") -_FORMULA_REF = re.compile(r"\{([^{}]*)\}") - - -def _quotes_balanced(s): - """An even number of `"` — the structural half of string support. - - Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it - contributes TWO characters. So a well-formed expression always has an even count and an - unterminated string always has an odd one. What a balanced pair MEANS is the engine's - business, exactly as with parentheses. - """ - return s.count('"') % 2 == 0 - - -#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that -#: silently never runs; `source` names where the run's subject comes from. -AUTOMATION_KINDS = {"instagram_profile"} -AUTOMATION_SOURCES = {"record_url_field", "self"} -MAX_AUTOMATION_SETTINGS = 12 - - -def _clean_automation(raw, valid_keys=None, flow_ids=None): - """Validate an `automation` config bag. Returns the clean dict, or None when there is - nothing valid to store (the column then renders as unconfigured — never invented). - - ⛔ WAVE 22 (contract C8, owner item 5) — NO FIELD WITHOUT A FLOW. With `flow_ids` given - (the WRITE path: grid_events / user_tables pass the tenant's automation-definition ids), - the bag MUST carry a `flowId` naming one of them — absent or naming a deleted flow is - refused, the same fail-closed direction as a formula ref that names no field. With - `flow_ids=None` (the READ path, `fields_from_workspace`) the law is NOT applied: a column - stored before the law must keep projecting — enforcement at read time would vaporise it, - which is the `_clean_formula` write/read split exactly. - """ - if not isinstance(raw, dict): - return None - kind = str(raw.get('kind') or '').strip() - if kind not in AUTOMATION_KINDS: - return None - source = str(raw.get('source') or 'record_url_field').strip() - if source not in AUTOMATION_SOURCES: - source = 'record_url_field' - out = {'kind': kind, 'source': source} - flow = str(raw.get('flowId') or '').strip()[:40] - if flow_ids is not None and (not flow or flow not in flow_ids): - return None - if flow: - out['flowId'] = flow - url_field = str(raw.get('urlField') or '').strip()[:80] - # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time - if url_field and (valid_keys is None or url_field in valid_keys): - out['urlField'] = url_field - settings = {} - for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]: - if isinstance(v, bool) or isinstance(v, (int, float)): - settings[str(k)[:40]] = v - elif isinstance(v, str): - settings[str(k)[:40]] = v[:200] - if settings: - out['settings'] = settings - return out - - -def _clean_formula(raw, valid_keys=None): - """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). - - Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` - refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a - wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a - Python mirror of the grammar: a second engine is the filter_sql-class drift risk. - Structure IS checked — charset, length, balanced parens, BALANCED QUOTES, well-formed - non-empty `{refs}` — and at WRITE time (`valid_keys` given) every ref must name a field this - table has, fail closed. At READ time refs are left alone: a referenced field deleted later - must blank the CELLS, not vaporise the column. - - ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so - `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the - fail-closed direction: the alternative is teaching this function where strings begin and - end, which is the second grammar the paragraph above refuses to write. - """ - if not isinstance(raw, str): - return None - s = raw.strip() - if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): - return None - depth = 0 - for ch in s: - if ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth < 0: - return None - if depth: - return None - if not _quotes_balanced(s): - return None - refs = _FORMULA_REF.findall(s) - leftover = _FORMULA_REF.sub("", s) - if "{" in leftover or "}" in leftover: # unbalanced / nested braces - return None - if any(not r.strip() for r in refs): # a `{}` ref names nothing - return None - if valid_keys is not None and any(r not in valid_keys for r in refs): - return None - return s - - -def _field_extras(saved, ftype): - """createdBy / permissions / format / scope — the validated passthrough the created strata - share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it - is carried so the client can gate its menus and the handler can enforce against it. - `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler - stamps it at create (cohort page only) and preserves it like createdBy; carried here so the - client can label the field, filtered OUT of other pages by fields_from_workspace.""" - out = {} - who = saved.get("createdBy") - if isinstance(who, str) and who.strip(): - out["createdBy"] = who.strip()[:80] - perms = _clean_permissions(saved.get("permissions")) - if perms: - out["permissions"] = perms - fmt = _clean_format(saved.get("format"), ftype) - if fmt: - out["format"] = fmt - if saved.get("scope") == "cohort": - out["scope"] = "cohort" - corrected_from = saved.get("labelCorrectedFrom") - correction_id = saved.get("labelCorrectionId") - if (isinstance(corrected_from, str) and corrected_from.strip() - and isinstance(correction_id, str) and correction_id.strip()): - out["labelCorrectedFrom"] = corrected_from.strip()[:120] - out["labelCorrectionId"] = correction_id.strip()[:180] - return out - - -#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). -#: -#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the -#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the -#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. -#: Putting it in the canonical contract would mean an always-present column that is empty for -#: everyone else, plus three consumers to keep in step for a value none of them can produce. -COHORT_COLUMN = 'cohorts' - - -def cohort_field(label='Locked views'): - """The derived membership column's descriptor. - - ⚠ WAVE 17 item 14 / C-STR — THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES - NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because - every stored view that shows or groups by this column names it by KEY, and every gate in - two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it - Cohort, but locked instead"), which is a change to what a reader sees — renaming the - identifiers would break saved views to change a word nobody reads. - - `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability - off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay - keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what - stops the column menu calling it a "source field" on that basis. - - ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another - column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains - "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse - than one that is absent. - """ - return { - 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', - 'default': False, 'filterable': False, 'derived': True, 'multi': True, - 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' - 'so grouping by this column puts a customer under EVERY view they are in — the ' - 'group counts therefore add up to more than the record count, which stays the ' - 'number of distinct customers. Read-only: membership changes only by adding or ' - 'removing customers on the locked view itself.', - } - - -def cohort_cells(cohorts, allowed_pids=None): - """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. - - Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, - never merged into the cached pool rows — those are shared across users, and stamping one - user's cohorts onto them would leak the membership to everybody else on the next render. - """ - cells = {} - for c in cohorts or []: - # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY - # cohort they are in, so it cannot also occur inside a name. Cohort names are free text - # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather - # than left to break the split silently — one group called "Q3 calls" and another called - # "west" would be two lists that do not exist. The cost is cosmetic and confined to the - # cell; the Cohort page still shows the name the user typed. - name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() - for pid in c.get('pids') or (): - if allowed_pids is not None and pid not in allowed_pids: - continue - cells.setdefault(pid, []).append(name) - return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} - - -def clean_measure_field(raw, offered): - """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. - - `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — - the same admission the condition builder uses, so a field can only name a measure this - caller could also filter by. Returns the canonical stored shape, or None (fail closed). - - `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: - the client keys editability off `source == 'overlay'` and the host accepts cell writes only - for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` - because the REPLACEMENT is the measure CONDITION with the same measure and window — the - governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. - """ - if not isinstance(raw, dict): - return None - key = str(raw.get("key") or "") - if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: - return None - spec = raw.get("measure") - if not isinstance(spec, dict): - return None - m = (offered or {}).get(spec.get("key")) - if not m: - return None # not admitted for this caller -> fail closed - window = _clean_window(spec.get("window")) - if window is None: - return None # a measure column with no period is not a column - mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" - return { - "key": key, - "label": str(raw.get("label") or m.get("label") or "Measure")[:120], - "type": mtype, - "source": "odoo", - "default": True, - "custom": True, - "derived": True, - "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, - "note": str(raw.get("note") or "")[:2000], - "measure": {"key": str(spec.get("key"))[:80], "window": window}, - } - - -def measure_fields_of(fields): - """The formula-measure columns among `fields` — the ones whose values the host must compute - per render (see `rows_from_pool`'s `derived`).""" - return [f for f in fields or [] if isinstance(f.get("measure"), dict)] - - -def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None): - """Overlay persisted notes/custom fields onto the immutable source-field contract. - - `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and - `default: False`, so it never displaces a column somebody already reads; the Fields menu is - where you turn it on. - - Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), - and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure - field was validated against the caller's measure OFFER when it was written - (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero - app-internal imports and cannot know the offer. A measure that has since become - unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value - time, never to an error. - - `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A - saved def carrying `scope` is emitted only when it matches — so a cohort-specific field - never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees - only unscoped (global) fields; base contract fields are never scoped. - - `fields_base` (wave 16 C-TOPIC) — the canonical contract to overlay onto. Absent = the - CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product - surface passes `product_fields()`. The workspace dict a caller hands in must already be the - matching table object's bucket — this function cannot tell a customer overlay from a - product one, which is exactly why the buckets are separate stores. - """ - saved = dict((workspace or {}).get("fields") or {}) - out = [] - base_fields = fields_base if fields_base is not None else FIELDS - base_keys = {field["key"] for field in base_fields} - for base in base_fields: - meta = saved.get(base["key"]) or {} - field = dict(base) - if isinstance(meta.get("note"), str): - field["note"] = meta["note"][:2000] - # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a - # number or date READS, per user. Rendering-only, so this is the whole acceptance. - fmt = _clean_format(meta.get("format"), base.get("type")) - if fmt: - field["format"] = fmt - # ⭐ W29-T83 — the saved COLUMN SUMMARY, the read half of the write door in - # `grid_events.field_upsert`. Without this the value round-trips into the store and is - # never served back, which looks exactly like a write that never happened - # ([[read-path-cannot-witness-write-path]]). Absent = whatever the contract declares. - if meta.get("agg") in FIELD_AGGS: - field["agg"] = meta["agg"] - # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved - # window/label override. Wave 6 deleted every preset member (the owner's - # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the - # measure_ path's twin for any future preset-carrying contract, and because deleting - # it would silently change what a re-added preset means. - if base.get("preset") and isinstance(base.get("measure"), dict): - saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} - window = _clean_window(saved_measure.get("window")) - if window is not None: - field["measure"] = {"key": base["measure"]["key"], "window": window} - if isinstance(meta.get("label"), str) and meta["label"].strip(): - field["label"] = meta["label"][:120] - out.append(field) - for key, field in saved.items(): - if key in base_keys or not isinstance(field, dict): - continue - if field.get("scope") and field.get("scope") != scope_key: - continue # a cohort-specific field on another page (wave-6 item 9) - if (str(key).startswith(MEASURE_FIELD_PREFIX) - and isinstance(field.get("measure"), dict)): - window = _clean_window(field["measure"].get("window")) - mkey = str(field["measure"].get("key") or "") - if window is None or not mkey: - continue - mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" - out.append({ - "key": str(key)[:80], - "label": str(field.get("label") or "Measure")[:120], - "type": mtype, - "source": "odoo", - "default": bool(field.get("default", True)), - "custom": True, - "derived": True, - "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, - "note": str(field.get("note") or "")[:2000], - "measure": {"key": mkey[:80], "window": window}, - **_field_extras(field, mtype), - }) - continue - if not str(key).startswith("custom_"): - continue - ftype = field.get("type") - if ftype not in CUSTOM_FIELD_TYPES: - continue - if ftype in READONLY_CUSTOM_TYPES: - # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort - # column's mechanism (source 'odoo' + derived) so the client never offers an - # editor and the host's overlay-write guard excludes them by construction. - # FILTERABLE since wave 6 (owner item 6): their values live client-side - # (formula computes over the row, created_time renders `_created`), this - # table's counts are client-mode, and the windowed count path never sees - # these tables — so the client engine answers them soundly. - entry = { - "key": str(key)[:80], - "label": str(field.get("label") or "Untitled")[:120], - "type": ftype, - "source": "odoo", - "derived": True, - "filterable": True, - "default": bool(field.get("default", True)), - "custom": True, - "note": str(field.get("note") or "")[:2000], - **_field_extras(field, ftype), - } - if ftype == "formula": - formula = _clean_formula(field.get("formula")) - if formula is None: - continue # a formula field without a formula is nothing - entry["formula"] = formula - out.append(entry) - continue - if field.get("source") != "overlay": - continue - options = (_clean_options(field.get("options")) - if ftype in ("select", "multiselect") else []) - if ftype in ("select", "multiselect") and not options: - # A select with no surviving choices can never hold a value. Dropping the COLUMN - # would lose the user's data; degrading it to text keeps every stored value - # readable and lets them re-add choices. - ftype = "text" - out.append({ - "key": str(key)[:80], - "label": str(field.get("label") or "Untitled")[:120], - "type": ftype, - "source": "overlay", - "default": bool(field.get("default", True)), - "custom": True, - # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a - # picker offering Average against a projection that only passes Sum through is the - # silent half of this feature. - "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, - "note": str(field.get("note") or "")[:2000], - **({"options": options} if ftype in ("select", "multiselect") else {}), - **(_choice_appearance(field, options) - if ftype in ("select", "multiselect") else {}), - # comma-joined SET semantics (the Cohorts column's contract): the row belongs to - # every member, groups count it under each, the toolbar count stays distinct. - **({"multi": True} if ftype == "multiselect" else {}), - **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), - **({"automation": _clean_automation(field.get("automation"))} - if ftype == "automation" and _clean_automation(field.get("automation")) else {}), - # ⭐ WAVE-27 item 13 (R13) — the code column's language rides the wire, because the - # highlighter is chosen per column and the client cannot infer a language from a - # string. Absent = `plain`, which is what an unconfigured code column renders as. - **({"code": _clean_code(field.get("code"))} - if ftype == "code" and _clean_code(field.get("code")) else {}), - **_field_extras(field, ftype), - }) - if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): - # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. - # A user table's field keys are slugged from whatever its creator typed, so a column - # literally called "Cohorts" produces the key `cohorts` — and appending here unguarded - # would put TWO fields with one key on the wire. The client indexes fields by key, so the - # duplicate does not error: it silently paints one column's values under the other's - # header. The user's own column wins; the derived one steps aside rather than shadowing it. - out.append(cohort_field()) - return out - - -def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): - """Map customer_data.pool() dicts -> the API row shape the grid expects: - pid + each Odoo field (numeric fields rounded, text/date passed through) + the - persisted external overlay. Mirrors the standalone API payload contract. - - `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than - reads off the pool row — today just the cohort column. A separate argument from `overlays` - on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is - never written back would make the dict mean two things. - """ - fields = fields or FIELDS - odoo_fields = [field for field in fields if field["source"] == "odoo"] - overlay_fields = [field for field in fields if field["source"] == "overlay"] - derived_keys = [field["key"] for field in fields if field.get("derived")] - overlays = overlays or {} - derived = derived or {} - out = [] - for r in pool_rows: - pid = r.get("pid") - # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the - # `created_time` field type renders, regardless of that field's own key. Not a Field: - # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride - # the same way: the Map VIEW's data, nullable, deliberately not a column. - row = {"pid": pid, "_created": r.get("_created") or "", - "lat": r.get("lat"), "lon": r.get("lon")} - for field in odoo_fields: - k = field["key"] - if field.get("derived"): - continue # not on the pool row — filled from `derived` below - v = r.get(k) - row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) - saved = overlays.get(str(pid), {}) or {} - for field in overlay_fields: - row[field["key"]] = saved.get(field["key"], "") - got = derived.get(pid) or {} - for k in derived_keys: - # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a - # text column is the question somebody will ask of it. - row[k] = got.get(k, "") - out.append(row) - return out - - -# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- -# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are -# VALUE-FREE (they legitimately carry no value and must never be dropped for it). -FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', - 'gt', 'gte', 'lt', 'lte', 'between', 'within', - # Wave 2026-08-02 (C-OPS): RANK operators — evaluated as a SET pass over the - # sibling-filtered domain by the client engine (useVisibleRows). The validator - # accepts them like any op (structural, not semantic); filter_sql REFUSES to - # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg / - # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a - # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100, - # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a - # measure-carrying column they rank the field's own derived values. - 'topN', 'bottomN', 'inTopPct', 'inBottomPct', - 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'} -#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not -#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean -#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. -COHORT_FIELD = '__cohort__' -#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. -#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: -#: a set op reaching a column leaf would fall through the client engine's switch to "no -#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. -COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} -#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: -#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the -#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. -COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} -#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. -MAX_COHORT_IDS = 20 - - -def parse_cohort_ids(value): - """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. - - Comma-separated in one string because `FilterRule.value` is what all four layers persist and - round-trip, and a one-element list is byte-identical to what the single-cohort leaf already - stored — so every shipped view parses with no migration. Safe because a cohort id is built - from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. - """ - out = [] - for raw in ('' if value is None else str(value)).split(','): - cid = raw.strip()[:120] - if not cid or cid in out: - continue - out.append(cid) - if len(out) >= MAX_COHORT_IDS: - break - return out -# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays -# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. -MAX_FILTER_DEPTH = 3 -MAX_FILTER_NODES = 100 # total nodes across the whole tree -MAX_FILTER_SIBLINGS = 50 # per level - - -#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by -#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored -#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already -#: need a gate to prevent. This validates SHAPE only. -WINDOW_MAX_N = 3650 - - -def _clean_window(raw): - """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. - - Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by - `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this - way keeps the grid module reusable and keeps one definition of what "last quarter" means. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if not isinstance(kind, str) or not kind or len(kind) > 40: - return None - out = {'kind': kind} - if raw.get('n') is not None: - try: - out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) - except (TypeError, ValueError): - return None - for side in ('from', 'to'): - if raw.get(side) not in (None, ''): - out[side] = str(raw[side])[:32] - return out - - -def _clean_rhs(raw, valid_keys): - """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. - - SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` - with the measure keys, exactly as it does for the left side), and a measure rhs must carry a - window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if kind not in ('field', 'measure', 'stat'): - return None - if kind == 'stat': - # A STATISTIC carries no column: the population is the comparand. Shape only — which - # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one - # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. - stat = raw.get('stat') - if not isinstance(stat, str) or not stat or len(stat) > 24: - return None - return {'kind': 'stat', 'stat': stat} - col = raw.get('colId') - if col not in valid_keys: - return None - out = {'kind': kind, 'colId': col} - if kind == 'measure': - window = _clean_window(raw.get('window')) - if window is None: - return None # a measure comparand with no period is not a question - out['window'] = window - return out - - -#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 -#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown -#: display means, so it is never stored. -#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to -#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the -#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing -#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live -#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The -#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on -#: READ forever, only ever WRITE 'chart'. -#: ⭐ WAVE-27 item 8 (owner ruling R2, contract C3): 'swipe' — a DECK of the records whose bound -#: single-select is EMPTY, triaged one at a time by swiping left or right into two of that -#: field's options. Landed here FIRST and in the same change as the client registry, which is -#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an -#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view -#: that silently reverts to a grid on the next read. -#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has -#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never -#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — -#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door -#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at -#: anything. The mirror is one name, and it is the half nobody could see was missing because -#: BOTH sides were individually consistent. -#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out -#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law -#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by -#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered -#: one is red too, so the hold cannot outlive its reason the way wave 27's did). -DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', - 'timeseries', 'catalog', 'swipe', 'form'} - -#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at -#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read -#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a -#: legal mode and this function had no branch for the key, so every spec a client sent was dropped -#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. -#: -#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a -#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — -#: so one tenant setting its token to another tenant's value would silently receive that tenant's -#: submissions. The token is minted server-side and lives in a bucket no client write can reach; -#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. -FORM_ACCESS = ('public', 'emails') -MAX_FORM_FIELDS = 60 -MAX_FORM_EMAILS = 200 -MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 -MAX_FORM_EMAIL = 254 - -#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the -#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing -#: from here is a question the builder cannot ask yet; a type wrongly present is a public door -#: writing values the column cannot mean (an `image` with no upload channel, a `json` document -#: typed into a text box, a `link` naming a record id a stranger guessed). -#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: -#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an -#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks -#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather -#: than re-derived ([[one-evaluator-per-question]]). -#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares -#: the two files name-for-name. -FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', - 'checkbox', 'phone', 'email', 'url', 'rating') - -#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in -#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the -#: expensive failure, not an odd address that gets in. -_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') - - -def _clean_form(raw, valid_keys): - """One form spec, fail-closed. Returns None when nothing is configured. - - ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the - builder let somebody arrange these questions, and sorting them by column order would silently - rearrange a live form every time a column was added (`_public_form` states the same rule from - the serving end). - - ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A - swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck - that can never write. A form is a LIST of questions — losing the column behind question three - costs the asker question three, and taking the whole form away because one field was deleted - would be a far larger loss than the one that happened. - """ - if not isinstance(raw, dict): - return None - fields, seen = [], set() - for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: - if k in valid_keys and k not in seen: - seen.add(k) - fields.append(k) - out = {} - if fields: - out['fields'] = fields - # A required flag on a question the form no longer asks is not a rule, it is a trap: the - # submitter can never satisfy it and the sentence names a field they cannot see. - req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] - if req: - out['required'] = req - for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), - ('submitLabel', MAX_FORM_SUBMIT)): - text = str(raw.get(key) or '').strip()[:cap] - if text: - out[key] = text - # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only - # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which - # is why `emails` is what gets written rather than a `public: false`. - if raw.get('access') == 'emails': - out['access'] = 'emails' - emails = [] - for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: - e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] - if _FORM_EMAIL.match(e) and e not in emails: - emails.append(e) - # Kept even while `access` is public: a person toggling the door open to test it and back - # again must not lose the list of people they typed. It is never served publicly. - if emails: - out['emails'] = emails - return out or None - - -#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a -#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same -#: number rather than a second opinion about it: a longer string cannot name a real option, and -#: a SHORTER cap here would silently refuse a binding to a legal one. -MAX_SWIPE_OPTION = 120 - -#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the -#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N / -#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops, -#: so an accepted save reads back byte-identically on both engines. -TS_BUCKETS = {'week', 'month', 'quarter', 'year'} -TS_MAX_LAST_N = 120 -TS_MAX_FIELDS = 12 -MAX_CALENDAR_METRICS = 4 -_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') - -#: C6-CATALOG (wave 18) — the catalog view's vocabulary and caps. types.ts mirrors every name -#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads -#: back identically on both engines. The code budget is the order-sensitive one — see -#: `_clean_catalogs`. -MAX_CATALOGS = 12 -MAX_CATALOG_PAGES = 40 -MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER -CATALOG_PAPERS = {'letter', 'a4', 'tabloid'} -CATALOG_ORIENTATIONS = {'portrait', 'landscape'} -CATALOG_QUALITIES = {'web', 'print'} -CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'} -CATALOG_COLS = (2, 3, 4) -CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80 -CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60 -_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') - -#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts -#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES — the C-DISP byte-identical law. -TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') -TS_MAX_CUSTOM_ROWS = 12 -TS_MAX_STYLES = 200 -#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client -#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole -#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. -_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') - -#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown -#: mode is still rejected rather than accidentally aliased. -_LEGACY_MODES = {'dashboard': 'chart'} - -#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's -#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a -#: kind the client cannot draw is worse than one that does not exist yet. -#: Wave-16 C-CHARTCAP: + 'table' — the group-by aggregate table (by-rep / by-BU / -#: top-customers, the third Sales block shape). Client renderer: DashboardView's -#: GroupTableView over salesParity.tableFromSpec. -CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} -CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} - -#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is -#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is -#: the picker's order; membership tests read it as a tuple perfectly well. -#: -#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` -#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): -#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole -#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum -#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are -#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both -#: would have to be the intersection of what each can express. -#: -#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` -#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, -#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would -#: give the product two words for one operation on two menus a click apart. -#: -#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column -#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. -#: -#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. -#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, -#: and must not grow a second spelling of it. -#: -#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by -#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to -#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. -FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') -MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop -MAX_CHART_TITLE = 60 - -#: Wave-9 I11 (contract C2) — chart customisation, host-validated. -#: -#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: -#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) -#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored -#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — -#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. -CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} -CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} -MAX_AXIS_LABEL = 40 -#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. -CHART_W_RANGE = (1, 12) -CHART_H_RANGE = (120, 800) - - -def _clean_chart(raw, valid_keys): - """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. - - A chart's `y` is optional (absent = count of rows, which is what "how many customers per - state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and - silently keeping it would put an empty card on the dashboard with no way to tell why. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if kind not in CHART_KINDS: - return None - x = raw.get('x') - if x not in valid_keys: - return None # dead category ref -> the chart goes, not the board - cid = raw.get('id') - if not isinstance(cid, str) or not cid.strip(): - return None # the client owns chart ids; an unidentified card - # cannot be edited or removed, so it must not persist - out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, - 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} - if raw.get('y') in valid_keys: - out['y'] = raw['y'] - else: - # no measurable column -> the only honest aggregation left is "how many rows" - out['agg'] = 'count' - title = raw.get('title') - if isinstance(title, str) and title.strip(): - out['title'] = title.strip()[:MAX_CHART_TITLE] - - # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── - # `splitBy` is the field whose values become the SERIES. It is deliberately not called - # `colorBy`: that name already means two other things here (`config.colorBy` = row - # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. - if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: - out['splitBy'] = raw['splitBy'] - # Stacking is only a question once there are series to stack, and only for the two kinds - # that can express it. Anywhere else it is dropped rather than stored as a lie the client - # would have to re-decide. - if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: - out['stacked'] = True - if raw.get('palette') in CHART_PALETTES: - out['palette'] = raw['palette'] - - axis = raw.get('axis') - if isinstance(axis, dict): - # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: - # two y-scales on one frame can manufacture any correlation you like by rescaling, and - # the honest alternatives are two charts, small multiples, or indexing to a common base. - # Ruled explicitly in contract C2 against the "Tableau versatility" brief. - clean_axis = {} - for side in ('x', 'y'): - spec = axis.get(side) - if not isinstance(spec, dict): - continue - one = {} - lab = spec.get('label') - if isinstance(lab, str) and lab.strip(): - one['label'] = lab.strip()[:MAX_AXIS_LABEL] - if spec.get('format') in CHART_FORMATS: - one['format'] = spec['format'] - if one: - clean_axis[side] = one - if clean_axis: - out['axis'] = clean_axis - - size = raw.get('size') - if isinstance(size, dict): - one = {} - for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): - try: - one[key] = max(lo, min(hi, int(size[key]))) - except (KeyError, TypeError, ValueError): - pass # a partial size is fine: the client defaults the missing axis - if one: - out['size'] = one - - # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD — the - # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed: - # a category column has no time dimension, and a stored period on it would promise a - # trend the TS channel must refuse. `span` is meaningful only beside `bucket`. - if isinstance(out.get('y'), str) and out['y'].startswith('measure_'): - if raw.get('bucket') in TS_BUCKETS: - out['bucket'] = raw['bucket'] - sp = raw.get('span') - if isinstance(sp, dict): - n = sp.get('lastN') - if (isinstance(n, int) and not isinstance(n, bool) - and 1 <= n <= TS_MAX_LAST_N): - out['span'] = {'lastN': n} - # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something — - # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta - # line). Anything else is a stored claim the renderer would have to re-refuse. - # Mirrors the client's cleanCharts rule key for key. - if raw.get('compare') == 'prior_year' and ( - out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')): - out['compare'] = 'prior_year' - return out - - -def _clean_catalog_page(raw, budget): - """C6-CATALOG — one page of a catalog. Returns `(page | None, codes_spent)`. - - `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a - page and not across the catalog: a product legitimately appears on a gallery page and again - in its section listing, and de-duplicating globally would silently delete the second - appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of - its last pages — never a random scatter, and never a page (a page with no products is a - heading the user can still see and fix). - """ - if not isinstance(raw, dict): - return None, 0 - page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX] - kind = raw.get('kind') - if not page_id or kind not in CATALOG_PAGE_KINDS: - return None, 0 - out = {'id': page_id, 'kind': kind} - for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX), - ('imageCode', CATALOG_CODE_MAX)): - v = raw.get(key) - if isinstance(v, str) and v: - out[key] = v[:cap] - products = raw.get('products') - if isinstance(products, list) and budget > 0: - clean_p, seen_p = [], set() - for c in products: - if not isinstance(c, str) or not c: - continue - c = c[:CATALOG_CODE_MAX] - if c in seen_p: - continue - seen_p.add(c) - clean_p.append(c) - if len(clean_p) >= budget: - break - if clean_p: - out['products'] = clean_p - layout = raw.get('layout') - if isinstance(layout, dict): - clean_l = {} - cols = layout.get('cols') - if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS: - clean_l['cols'] = cols - # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by - # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all). - # So only the opt-OUT is storable for the first two and only the opt-IN for the third — - # a second spelling of a default is how a round trip starts churning. - if layout.get('showPack') is False: - clean_l['showPack'] = False - if layout.get('showColor') is False: - clean_l['showColor'] = False - if layout.get('showPrice') is True: - clean_l['showPrice'] = True - if clean_l: - out['layout'] = clean_l - return out, len(out.get('products') or ()) - - -def _clean_catalogs(raw, valid_keys): - """C6-CATALOG (wave 18) — `display.catalogs`, fail-closed. Returns a list or None. - - A catalog is a PRINT artifact, so the two structural keys that decide how it paginates - (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with - no size is not a smaller catalog, it is an unrenderable one. Everything else follows the - house rules — unknown keys dropped, per-entry drops never cost the neighbours, empty - sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and - an empty one are not two spellings of the same nothing. - - `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU / - pack / colour, and `product_data` carries no pack or colour of its own — the user binds - custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same - split `dateField`/`stackField` already run. - """ - if not isinstance(raw, list): - return None - out = [] - for c in raw: - if len(out) >= MAX_CATALOGS: - break - if not isinstance(c, dict): - continue - cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX] - name = c.get('name') - # An EMPTY name is legal (the user cleared the box and will type again) — an ABSENT one - # is a malformed record. The `tsRows` label rule, same reasoning. - if not cat_id or not isinstance(name, str): - continue - cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]} - cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter' - cat['orientation'] = (c['orientation'] - if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait') - if c.get('quality') in CATALOG_QUALITIES: - cat['quality'] = c['quality'] - brand = c.get('brand') - if isinstance(brand, dict): - clean_b = {} - for k in ('primary', 'accent'): - v = brand.get(k) - if isinstance(v, str) and _HEX6.match(v): - clean_b[k] = v - company = brand.get('company') - if isinstance(company, str) and company: - clean_b['company'] = company[:CATALOG_NAME_MAX] - # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside - # print CSS is exactly the tokens-not-values rule this contract carries. - logo = brand.get('logo') - if isinstance(logo, str) and logo: - clean_b['logo'] = logo[:CATALOG_CODE_MAX] - if clean_b: - cat['brand'] = clean_b - binds = c.get('fields') - if isinstance(binds, dict): - clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price') - if binds.get(k) in valid_keys} - if clean_bind: - cat['fields'] = clean_bind - pages, budget = [], MAX_CATALOG_CODES - raw_pages = c.get('pages') - if isinstance(raw_pages, list): - for p in raw_pages: - if len(pages) >= MAX_CATALOG_PAGES: - break - page, spent = _clean_catalog_page(p, budget) - if page is None: - continue - budget -= spent - pages.append(page) - # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog - # starts in, and dropping the key would make "new" and "corrupt" the same wire value. - cat['pages'] = pages - out.append(cat) - return out or None - - -def _clean_display(raw, valid_keys): - """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. - - `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode - must be a known non-grid mode (grid is the absent default, so storing it would be a second - way to say nothing); every field ref must name a field this table has (a ref to a deleted - field is DROPPED and the client falls back to its per-mode default); unknown keys are - dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family - stack, map a single-select to colour by and a numeric to size by — is the client's - business: it is the only layer that renders them, and a wrong-typed ref degrades to that - surface's default rather than to an error (the `_clean_window` split). - - Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and - dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — - never the whole array, because losing one column should not cost the user a dashboard they - spent time building. - - W33-T45 (contract C1, ruling R5) adds `published` + `publishAccess`. ⚠ THE HALF OF THE ROUND - TRIP THIS FUNCTION CANNOT ENFORCE: `customer-grid/types.ts::cleanDisplay` is a SECOND - normalizer, in the browser, which rebuilds the config key by key on every autosave and drops - anything it does not name. A key accepted here and unknown there dies on the next column - resize — silently, because the client then POSTs the stripped config and the host REPLACES - the stored one. "The host accepts it" is half a round trip; that file is the other half. - """ - if not isinstance(raw, dict): - return None - mode = raw.get('mode') - if mode not in DISPLAY_MODES or mode == 'grid': - return None - # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown - # mode is still rejected rather than accidentally aliased into a real one. Every already - # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. - mode = _LEGACY_MODES.get(mode, mode) - out = {'mode': mode} - for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): - if raw.get(ref) in valid_keys: - out[ref] = raw[ref] - # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── - # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped — the new - # standardized default — so storing True would be a second way to say nothing (the same - # rule that keeps mode:'grid' out of the store). - if raw.get('kanbanClamp') is False: - out['kanbanClamp'] = False - if raw.get('calendarMode') in ('records', 'summary'): - out['calendarMode'] = raw['calendarMode'] - metrics = raw.get('calendarMetrics') - if isinstance(metrics, list): - clean_m, seen_m = [], set() - for m in metrics[:MAX_CALENDAR_METRICS]: - # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the - # user the summary card they configured around it. - if not isinstance(m, dict): - continue - mid = str(m.get('id') or '')[:40] - if (not mid or mid in seen_m or m.get('field') not in valid_keys - or m.get('agg') not in CHART_AGGS): - continue - seen_m.add(mid) - clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']}) - if clean_m: - out['calendarMetrics'] = clean_m - if raw.get('tsBucket') in TS_BUCKETS: - out['tsBucket'] = raw['tsBucket'] - span = raw.get('tsSpan') - if isinstance(span, dict): - clean_span = {} - n = span.get('lastN') - if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N: - clean_span['lastN'] = n - else: - f, t = span.get('from'), span.get('to') - f = f if isinstance(f, str) and _ISO_DAY.match(f) else None - t = t if isinstance(t, str) and _ISO_DAY.match(t) else None - if f and t and f > t: - f, t = t, f - if f: - clean_span['from'] = f - if t: - clean_span['to'] = t - if clean_span: - out['tsSpan'] = clean_span - ts_fields = raw.get('tsFields') - if isinstance(ts_fields, list): - clean_f, seen_f = [], set() - for k in ts_fields[:TS_MAX_FIELDS]: - if k in valid_keys and k not in seen_f: - seen_f.add(k) - clean_f.append(k) - if clean_f: - out['tsFields'] = clean_f - # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ──────────────────────── - deltas = raw.get('tsDeltas') - if isinstance(deltas, list): - clean_d, seen_d = [], set() - for d in deltas: - if d in TS_DELTA_KINDS and d not in seen_d: - seen_d.add(d) - clean_d.append(d) - if clean_d: - out['tsDeltas'] = clean_d - # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the - # literal opt-IN (absent = off) — the kanbanClamp asymmetry, one per direction. - if raw.get('tsGridlines') is False: - out['tsGridlines'] = False - if raw.get('tsSparkline') is True: - out['tsSparkline'] = True - rows = raw.get('tsRows') - if isinstance(rows, list): - clean_r, seen_r = [], set() - for r in rows[:TS_MAX_CUSTOM_ROWS]: - if not isinstance(r, dict): - continue - rid = str(r.get('id') or '')[:40] - r_kind = r.get('kind') - if not rid or rid in seen_r or r_kind not in ('note', 'formula'): - continue - label = r.get('label') - if not isinstance(label, str): - continue # ABSENT label = malformed; an EMPTY one is a legal spacer row - # (GRID's dated asymmetry amendments, 2026-08-02) - one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} - if r_kind == 'formula': - expr = r.get('expr') - if (isinstance(expr, str) and expr.strip() - and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): - one['expr'] = expr - # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row - # would delete the user's label to punish their arithmetic (GRID's dated - # asymmetry amendment; the calendarMetrics per-entry-drop precedent). - seen_r.add(rid) - clean_r.append(one) - if clean_r: - out['tsRows'] = clean_r - styles = raw.get('tsStyles') - if isinstance(styles, dict): - clean_s = {} - for s_key, s_val in styles.items(): - if len(clean_s) >= TS_MAX_STYLES: - break # capped, not truncated silently: the gate names this - if not isinstance(s_key, str) or not s_key or len(s_key) > 96: - continue # key = rowId or "rowId:colKey" — the client's grammar - if not isinstance(s_val, dict): - continue - one = {} - if s_val.get('bold') is True: - one['bold'] = True - if s_val.get('line') is True: - one['line'] = True - if one: - clean_s[s_key] = one - if clean_s: - out['tsStyles'] = clean_s - charts = raw.get('charts') - if isinstance(charts, list): - clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] - # de-dupe by id: two cards sharing an id are one card as far as the client's keyed - # render is concerned, and the second would silently shadow the first - seen, uniq = set(), [] - for c in clean: - if c['id'] in seen: - continue - seen.add(c['id']) - uniq.append(c) - if uniq: - out['charts'] = uniq - # ── C6-CATALOG (wave 18) ───────────────────────────────────────────────────────────── - catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) - if catalogs: - out['catalogs'] = catalogs - # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── - # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that - # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. - # Those are LISTS of independent cards, so losing one entry costs the user one card. This is - # a single three-part BINDING: a swipe view holding a fieldKey with one option, or two - # options and no field, is not a degraded swipe view — it is a deck that can never write - # anything, rendered as though it were configured. Dropping the key entirely puts the view - # back in its honest unconfigured state, which is the one state the client has a UI for. - # - # ⚠ What this CANNOT check, deliberately, and why the client must: whether `fieldKey` names - # a SELECT, and whether the two options are still in that select's vocabulary. `valid_keys` - # is a key set, and the docstring above draws this exact line — "what each mode MEANS ... is - # the client's business". So SwipeView owns three losses this function is blind to (field - # deleted, field retyped away from select, option removed) and must SHOW each one rather - # than fall back to the first option, per the `viewModes.tsx` house rule. - swipe = raw.get('swipe') - if isinstance(swipe, dict): - f_key = swipe.get('fieldKey') - left, right = swipe.get('leftOption'), swipe.get('rightOption') - ok = (f_key in valid_keys - and isinstance(left, str) and isinstance(right, str)) - if ok: - left, right = left.strip()[:MAX_SWIPE_OPTION], right.strip()[:MAX_SWIPE_OPTION] - # Both non-empty, and DISTINCT: one option on both sides is a deck whose two - # gestures do the same thing, which is two spellings of one state (the - # `kanbanClamp` law) wearing a control that promises a choice. - if left and right and left.casefold() != right.casefold(): - out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} - # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. - form = _clean_form(raw.get('form'), valid_keys) - if form: - out['form'] = form - # ── ⭐⭐ W33-T45 / CONTRACT C1 / RULING R5 (owner item 8b): IS THIS INTERFACE PUBLISHED ──── - # - # ⛔ TWO KEYS LIVE HERE AND TWO DELIBERATELY DO NOT. The `published` flag and the sharer's - # `public | password` choice are DISPLAY state — the view says what it is, the client renders - # a badge from it, and it travels with the view like every other key in this dict. The SECRET - # TOKEN and the PASSPHRASE HASH do not: they live in the server-only bucket, exactly as - # `routes_forms.py`'s `TOKENS_KEY` holds the form token. `config.display` is echoed back to - # every user who can open the view, so a token in here is a token published to the audience - # the password was meant to exclude. `_clean_form` above carries the same rule and the same - # reason; this is the second door, not a new one. - # - # ⛔ AND THE COERCION IS FAIL-CLOSED, WHICH IS WHY THIS IS NOT A BARE ALLOWLIST. A plain - # allowlist drops an unrecognised `publishAccess` and KEEPS `published: True` — leaving a - # published view with no stated access, i.e. a third state neither the ruling nor the client - # has a meaning for, on the one key where guessing wrong publishes a tenant's data to the - # open internet. So: a published view ALWAYS carries an access, and anything that is not the - # literal `'public'` reads as `'password'`. The unpublished case stores nothing at all — - # absent means unpublished, and a `published: False` would be the second way to say nothing - # that `kanbanClamp` and `mode: 'grid'` are both here to forbid. - if raw.get('published') is True: - out['published'] = True - out['publishAccess'] = 'public' if raw.get('publishAccess') == 'public' else 'password' - return out - - -#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). -#: -#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each -#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. -#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort -#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated -#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of -#: what a view IS: keeping it out of the view config means duplicating or exporting a view does -#: not drag a folder reference along with it. One map, one home, both surfaces. -FOLDER_SURFACES = {"views", "cohorts"} -MAX_FOLDERS = 60 -MAX_FOLDER_NAME = 80 - -#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. -#: -#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already -#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, -#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the -#: client cannot render can never reach the store. -#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — -#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional -#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set -#: without the matching client geometry: an unknown shape falls back to the default folder mark, -#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. -FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} -#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the -#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. -FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} -FOLDER_ICON_DEFAULT_TONE = "grey" - - -#: Wave-9 I17 (contract C4) — who may EDIT a saved view. -#: -#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today -#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one -#: user's views are invisible to every other user and "collaborative" has nothing to act on -#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable -#: and fail-closed now, so that when shared views land the permission does not need a data -#: migration and no saved view is retro-restricted. It does NOT make anything shared, and -#: nothing in the app currently reads it to grant or deny cross-user access. -#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. -VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} -MAX_VIEW_USERS = 50 - - -def clean_view_permissions(raw, default, known_users=None): - """{edit, users?} — fail-closed on both halves. - - `default` is supplied by the CALLER because it splits by path, and that split is a - permission rule rather than a formatting one: absent on a view that already exists means a - pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a - silent takeaway), while absent on CREATE must be 'personal' (a new view must never be - anyone-can-edit purely by omission). - - `known_users` (when given) is the real account list: an unknown name is DROPPED, and an - 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. - """ - mode = (raw or {}).get('edit') if isinstance(raw, dict) else None - if mode not in VIEW_EDIT_MODES: - mode = default if default in VIEW_EDIT_MODES else 'personal' - if mode != 'users': - return {'edit': mode} - names, seen = [], set() - for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: - u = str(u or '').strip() - if not u or u.lower() in seen: - continue - if known_users is not None and u not in known_users: - continue # fail-closed: a name we cannot resolve grants nothing - seen.add(u.lower()) - names.append(u) - if not names: - return {'edit': 'personal'} # an empty grant is NOT "everyone" - return {'edit': 'users', 'users': names} - - -def clean_folder_icon(raw): - """{shape, tone} or None. Fail-closed on both halves, independently. - - A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather - than losing the icon entirely — losing a user's pick because one half was wrong is the kind - of silent data loss the folder events already avoid elsewhere. - """ - if not isinstance(raw, dict): - return None - shape = raw.get("shape") - if shape not in FOLDER_ICON_SHAPES: - return None - tone = raw.get("tone") - return {"shape": shape, - "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} - - -def clean_folders(raw): - """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" - out = {} - for surface in FOLDER_SURFACES: - items, seen = [], set() - for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:80] - name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] - if not fid or not name or fid in seen: - continue # an unidentified or unnamed folder cannot be shown or edited - seen.add(fid) - try: - order = int(f.get("order", len(items))) - except (TypeError, ValueError): - order = len(items) - row = {"id": fid, "name": name, "order": order} - icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark - if icon: - row["icon"] = icon - items.append(row) - items.sort(key=lambda x: x["order"]) - for i, f in enumerate(items): - f["order"] = i # re-index so `order` is always dense and total - if items: - out[surface] = items - return out - - -#: ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, raised by SESSION C as ASK C-16) — "FILED AT ROOT". -#: -#: ⛔ THE DEFECT IS THAT ROOT WAS REPRESENTED BY *ABSENCE*, AND ABSENCE CANNOT HOLD TWO FACTS. -#: "this arrived by grant and was never filed" and "the receiver deliberately dragged this OUT of -#: the Shared group" were the same stored state — nothing — so the client had to GUESS, and -#: `folders.ts::groupByFolder` guessed "Shared". That is why only folder→folder moves appeared to -#: work: **the root bucket was unreachable for a shared view by construction.** -#: -#: ⚠ A RESERVED FOLDER ID, NOT A NEW FIELD, deliberately. The placement map is `{itemId: folderId}` -#: and every reader on both sides already understands it; a parallel "filedAtRoot" set would be a -#: second source of truth for one question, and the two would disagree the first time one of them -#: was written without the other. This id names no folder BY DESIGN and is therefore exempt from -#: the folder-exists test below — it is the one value that means "no folder, on purpose". -#: ⚠ Spelled `ROOT_FOLDER_ID` on the client (`customer-grid/folders.ts`, C's file). Two spellings -#: of one constant is [[a-constant-two-features-share]]; `verify_folders`/`verify_api` assert they -#: agree rather than a comment asking nicely. -ROOT_PLACEMENT = "__root__" - - -def clean_item_folders(raw, folders, valid_ids): - """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. - - This is what makes a deleted folder's contents fall back to the root rather than vanish: - nothing stores "this item is in no folder", so an unresolvable placement simply disappears - and the item renders at the top level. Same for an item that was deleted elsewhere — its - stale placement can never resurrect it, because the sidebars render ITEMS and consult this - map, never the other way round. - - ⭐⭐ WAVE 32 — THE PARAGRAPH ABOVE STATES THE FEATURE AND THE BUG IN ONE SENTENCE, and it took - owner item 20 to notice they were the same mechanism. *"Nothing stores 'this item is in no - folder', so an unresolvable placement simply disappears"* is exactly right for a DELETED FOLDER - (its contents should fall to the root) and exactly wrong for a SHARED VIEW (falling back means - falling back INTO the Shared group, which is where it started). `ROOT_PLACEMENT` is the value - that survives this function so the second case can be said out loud. - """ - out = {} - for surface in FOLDER_SURFACES: - fids = {f["id"] for f in (folders or {}).get(surface, [])} - ok = {} - for item_id, fid in ((raw or {}).get(surface) or {}).items(): - if not isinstance(item_id, str) or not isinstance(fid, str): - continue - # ⛔ `fid == ROOT_PLACEMENT` FIRST, and it is NOT in `fids` — it names no folder, which - # is the whole point. Without this clause the value is written by `item_move` and - # scrubbed here on the way back out, so the mark would be stored and instantly lost: - # the two halves are ONE change and shipping either alone is worse than shipping - # neither ([[lost-write-looks-like-failed-read]]). - if item_id in (valid_ids or {}).get(surface, ()) and (fid == ROOT_PLACEMENT - or fid in fids): - ok[item_id[:120]] = fid - if ok: - out[surface] = ok - return out - - -def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): - """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). - - Returns a clean tree of leaf conditions ({colId, op, value, value2}) and - groups ({conj, children}). Module-agnostic on purpose: any module embedding - the grid validates its own view state through this one function. - - Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — - the same contract the rest of the view sanitiser follows, so one bad rule can - never cost a user their whole saved view. Depth, per-level width and total - node count are all capped: the tree is re-evaluated for every row on every - render, so an unbounded structure would be a persistent client-side DoS. - Empty groups are dropped (they carry no meaning once persisted). - - `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is - DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a - condition that can only match nothing, so `List is not [deleted]` would show an empty table - forever with no way to tell why. `None` means this host has no cohorts, and then every - cohort leaf is dropped: fail-closed, like every other unknown key. - """ - if budget is None: - budget = [MAX_FILTER_NODES] - out = [] - for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: - if budget[0] <= 0: - break - if not isinstance(node, dict): - continue - if isinstance(node.get('children'), list): # a condition GROUP - if depth >= MAX_FILTER_DEPTH: - continue # too deep -> drop - budget[0] -= 1 - children = clean_filter_tree(node['children'], valid_keys, - depth + 1, budget, cohort_ids) - if children: - out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', - 'children': children}) - continue - if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf - op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) - named = parse_cohort_ids(node.get('value')) - # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT - # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to - # `is none of [A]` would show every row in B under a count nobody would doubt. This - # is the same all-or-nothing the single-cohort leaf already had, extended to a set. - if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): - budget[0] -= 1 - out.append({'colId': COHORT_FIELD, 'op': op, - 'value': ','.join(named), 'value2': ''}) - continue - if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: - budget[0] -= 1 - # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the - # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client - # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently - # stop filtering and show every row instead of the zero-revenue ones. - val, val2 = node.get('value'), node.get('value2') - leaf = {'colId': node['colId'], 'op': node['op'], - 'value': ('' if val is None else str(val))[:500], - 'value2': ('' if val2 is None else str(val2))[:500]} - # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two - # extra members: a stable client-generated `id`, which is how the server's answer - # finds its way back to the condition that asked (positional matching silently - # re-associates every answer the moment a user deletes a condition), and the - # `window`. Emitted ONLY when the input has them — a column condition's cleaned - # shape is unchanged, so every persisted view deserialises byte-identically and - # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by - # exact structural comparison). - rid = node.get('id') - if rid not in (None, ''): - leaf['id'] = str(rid)[:64] - window = _clean_window(node.get('window')) - if window is not None: - leaf['window'] = window - # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted - # only when the input has them, so a plain column condition's cleaned shape is - # byte-identical to what it always was and `clean_filter_tree` stays idempotent - # (verify_filter_engine.py asserts that by exact structural comparison). Drop the - # carry-through and the next autosave silently strips a date condition back to a - # bare comparison against an empty value — i.e. back to INACTIVE. - date_window = _clean_window(node.get('dateWindow')) - if date_window is not None: - leaf['dateWindow'] = date_window - mode = node.get('dateMode') - # SHAPE only. An unrecognised mode survives here and is refused by - # `windows.resolve_anchor`, which returns None and makes the condition match - # NOTHING — the same split as `_clean_window`, and the reason this module can stay - # free of the date vocabulary it would otherwise have to keep in step. - if isinstance(mode, str) and 0 < len(mode) <= 40: - leaf['dateMode'] = mode - rhs = _clean_rhs(node.get('rhs'), valid_keys) - if rhs is not None: - leaf['rhs'] = rhs - out.append(leaf) - return out - - -def _default_view_config(fields): - # ⭐⭐ W30-T41's SERVER HALF (F's ask F-1, answered by D — this file is D's fence). - # - # ⛔ THE SECOND ARM USED TO BE `or field["source"] == "overlay"`, AND IT SWALLOWED THE FIRST - # ONE FOR EVERY CONNECTED COLUMN. `user_tables._clean_field` stamps `source: "overlay"` on - # every `ut_` field, so on an Odoo grid the arm was true for ALL of them and `default: False` - # meant nothing: `odoo_id`, `state`, `customer_link` and `partner_id` opened SHOWN however - # they were declared. The exception had become the rule ([[fallback-that-became-the-rule]]), - # and it is the same predicate `useGridColumns.isDefaultVisible` carried on the client. - # - # ⚠ AND THE TWO HALVES MUST MOVE TOGETHER, which is why this is not cosmetic. `CustomerGrid` - # compares the stored view against its own `defaultViewConfig` by JSON equality; with the - # client fixed (T41) and this left alone, the system view would differ from the client's - # default on every render — a view that looks permanently dirty and autosaves forever, which - # is the failure `verify_filter_engine`'s key-ORDER check exists to prevent, one level down. - # - # ⚠ A USER-CREATED COLUMN IS UNAFFECTED, and that is why the fix is a DELETION rather than a - # carve-out for the four Odoo keys: it carries no `default` key at all, so `is not False` - # keeps it visible. On the main Customer grid exactly one field moves — `notes`, which asks - # to be hidden in its own declaration and was being shown against it. - shown = [field["key"] for field in fields if field.get("default") is not False] - hidden = [field["key"] for field in fields if field["key"] not in shown] - return { - # `filters` is the ROOT of the filter tree: leaf conditions and/or nested - # condition groups ({conj, children}); `filterConj` joins the root level. - "filters": [], "filterConj": "and", - "sorts": [], "groupBy": None, "colorBy": None, - "rowHeightMode": "short", "order": shown + hidden, "visible": shown, - "widths": {}, "memberPids": [], - } - - -#: Wave 17 R1 / C-LOCKV — the `kind` a PROJECTED locked view wears. A cohort is not a separate -#: kind of object any more: it is a saved view whose rows are a hand-curated set. -LOCKED_VIEW_KIND = 'locked' - - -def locked_view_projection(entry, base_config): - """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV). - - ⛔ THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the - view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the - named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do - all the work with no second mechanism. Membership itself is NEVER copied in here — it stays - in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped - collection inside a client-writable `config` is deleted by the next autosave (see the - contract's reason 2). - - ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set: - these views are ordinary in every respect the owner asked for — reorder, folder, sort, - filter, change display mode. The lock mark in the rail is driven by `kind`. - """ - return { - 'id': entry['id'], - 'name': entry.get('name') or entry['id'], - 'kind': LOCKED_VIEW_KIND, - 'config': {**base_config, 'cohortLock': entry['id']}, - } - - -#: ⭐ WAVE-27 item 27 (owner ruling R8) — the IG "Overview" view's CURATED COLUMNS, in the -#: owner's own order: handle, followers, engagement, location, last enriched. -#: -#: Written as candidates rather than as a requirement. The template registry REFUSES a template -#: whose columns the target lacks (`view_templates.missing_columns`) because applying one writes -#: the user's own views and a filter on a missing column silently WIDENS. This view is INJECTED, -#: not applied, and it filters nothing — so the proportionate rule is the opposite one: take the -#: columns the table has, in this order, and skip the rest. An IG database that predates a -#: column simply shows the other four. -#: -#: ⚠ `location_guess` is SESSION B's item-16 column and may not exist yet. That is exactly why -#: this list is intersected rather than asserted: a hard requirement here would make the whole -#: view vanish (or the assembly refuse) on every tenant until B lands, and then appear by -#: surprise. `profile_url` closes the list as the click-through, which is what makes the view -#: usable rather than merely informative. -IG_OVERVIEW_COLUMNS = ('handle', 'full_name', 'followers', 'avg_engagement', - 'location_guess', 'enriched_at', 'profile_url') - -#: The id is PINNED, the `view_templates` discipline: re-assembling must update the same view -#: rather than mint "Overview 2". It also lets a user's own edits overlay it through the saved -#: -config loop below, exactly as a cohort projection does. -IG_OVERVIEW_ID = 'tpl_overview' - -#: How this function recognises an IG preset database WITHOUT importing the engine: two of the -#: profile preset columns is a stronger signal than any single one (a hand-made table could -#: plausibly own a column called `followers`; owning `followers` AND `avg_engagement` AND -#: `handle` is the preset set). `core/` must stay importable without the API layer, so this -#: mirrors `user_tables.PROFILE_PRESET_KEYS` the way that module mirrors the engine's. -_IG_SIGNATURE = ('handle', 'followers', 'avg_engagement') - - -def _ig_overview_view(fields, base): - """R8's curated Overview, or None when this table is not an Instagram one.""" - keys = {f['key'] for f in fields} - if not all(k in keys for k in _IG_SIGNATURE): - return None - visible = [k for k in IG_OVERVIEW_COLUMNS if k in keys] - return { - 'id': IG_OVERVIEW_ID, - 'name': 'Overview', - 'kind': 'system', - # NOT `locked`. The system view is locked because it is the identity of the table ("show - # me everything"); this one is a STARTING LAYOUT, and R8 calls it curated rather than - # fixed. A user who wants a sixth column should get one. - 'note': 'The five things worth seeing first on a creator. Sorted by reach.', - 'config': { - **dict(base), - 'visible': visible, - 'order': visible + [k for k in (f['key'] for f in fields) if k not in visible], - 'sorts': ([{'colId': 'followers', 'dir': 'desc'}] - if 'followers' in keys else []), - }, - } - - -def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None, - view_order=None): - """Convert legacy list formulas into the shared serializable SavedView contract. - - `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the - product surface). The ID stays "all-customers" on every topic — the client pins it - (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork - that contract for a label's sake. - - `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose - id IS the cohort id — so every stored reference to that id (a `cohortLock` on another view, - an `is part of` condition, a folder placement) keeps pointing at the same thing and no - rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the - `list:` views have always used, which is what gives a locked view its own sort, filter, - columns and display mode with no new storage.""" - base = _default_view_config(fields) - views = [{ - "id": "all-customers", "name": system_name, "kind": "system", - "locked": True, "config": dict(base), - }] - # ⭐ WAVE-27 item 27 (R8) — the IG Overview, ABOVE All records. - # - # ⚠ INJECTED, not seeded into the store, and that is what makes "existing IG databases gain - # it too" true with no migration and no write on a read path. It is the same mechanism the - # system view above has always used; the pinned id means a user's own edits overlay it - # through the saved-config loop below rather than forking a second view. - _overview = _ig_overview_view(fields, base) - if _overview: - views.insert(0, _overview) - op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", - "contains": "contains"} - for name, definition in (defs or {}).items(): - filters = [] - for rule in definition.get("rules") or []: - if rule.get("field") not in {field["key"] for field in fields}: - continue - filters.append({ - "colId": rule["field"], - "op": op_map.get(rule.get("op"), "eq"), - "value": str(rule.get("value") if rule.get("value") is not None else ""), - }) - sort = str(definition.get("sort") or "") - sorts = ([{"colId": sort.lstrip("-"), - "dir": "desc" if sort.startswith("-") else "asc"}] - if sort.lstrip("-") in {field["key"] for field in fields} else []) - views.append({ - "id": "list:" + str(name), - "name": str(name), - "kind": "list", - "note": str(definition.get("note") or ""), - "config": { - **base, "filters": filters, "sorts": sorts, - "memberPids": [int(pid) for pid in definition.get("members") or [] - if isinstance(pid, int) or str(pid).isdigit()], - }, - }) - # Wave 17 R1 — the cohorts, as ordinary views. Appended BEFORE the saved-config overlay - # below so a user's own edits to a locked view (its sort, its columns, its display mode) - # land on the projection instead of creating a second row with the same id. - for _entry in (locked_lists or []): - if isinstance(_entry, dict) and _entry.get('id'): - views.append(locked_view_projection(_entry, base)) - index = {view["id"]: i for i, view in enumerate(views)} - for view_id, saved in (saved_views or {}).items(): - if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): - continue - clean = dict(saved) - clean["id"] = str(view_id) - if view_id in index: - views[index[view_id]] = clean - else: - views.append(clean) - # ⛔ WAVE 17 R1 — RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected - # view with its saved record, and a saved record that omits `cohortLock` would hand back a - # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the - # client rebuilds `config` on every autosave (a column resize is enough), and the lock is - # identity here, not something the browser is the source of truth for. Read-side rather than - # write-side-only on purpose — this also repairs any record already written by another path. - _locked_ids = {e['id']: e for e in (locked_lists or []) - if isinstance(e, dict) and e.get('id')} - if _locked_ids: - for _v in views: - _entry = _locked_ids.get(_v.get('id')) - if not _entry: - continue - _v['kind'] = LOCKED_VIEW_KIND - _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']} - # One thing, one name: the cohort store owns it (the rename event routes there), so - # a stale `name` on the saved record can never fork into a second title. - _v['name'] = _entry.get('name') or _v['id'] - # ── ⭐ WAVE-27 item 5, contract C7: the PER-USER VIEW ORDER ─────────────────────────────── - # - # `view_order` is a list of view ids this user dragged into place. Applied LAST, over the - # finished list, so it reorders whatever the assembly produced without having to know how any - # of it got there (system, list:, cohort projection, saved, injected Overview). - # - # ⛔ THE SYSTEM VIEW STAYS AT INDEX 0 (C7), and it is re-pinned here rather than trusted to - # sort correctly: `all-customers` is the client's landing default and one of its - # UNDELETABLE_VIEW_IDS, so a stored order that happened to omit it — or list it third — - # would move the rail's home row. ⚠ R8's Overview is the ONE thing allowed above it, because - # the owner put it there; it is re-pinned with the system view so a drag cannot bury it - # either. Both are facts about the table rather than the user's arrangement of it. - # - # ⚠ UNKNOWN IDS APPEND IN SERVER ORDER (C7). A view created since this order was stored, or - # one shared to this user yesterday, must APPEAR — dropping it would make sharing look - # broken, and the failure would be invisible to whoever shared it. Ids in the stored order - # that no longer resolve are simply skipped. - if view_order: - _rank = {vid: i for i, vid in enumerate(view_order) if isinstance(vid, str)} - _pinned = [v for v in views if v.get('id') in (IG_OVERVIEW_ID, 'all-customers')] - _rest = [v for v in views if v.get('id') not in (IG_OVERVIEW_ID, 'all-customers')] - # A stable sort over a rank that DEFAULTS TO THE END keeps unranked views in their - # server order behind the ranked ones, rather than interleaving them by accident. - _rest.sort(key=lambda v: _rank.get(v.get('id'), len(_rank) + 1)) - views = _pinned + _rest - return views - - -def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, - fields_base=None, with_cohorts=True): - """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE - projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`). - - ⛔ WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no - `storageKey` — and the client validator (`fetchWorkspace`) requires one, so the standalone - shell silently discarded the whole workspace: saved views never rendered and `cohortMode` - never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline - projection into the route would have re-created the same drift one wave later; extracting it - means the wire can only be one thing. - - Returns `(workspace, fields, views, cohort_lists)` — the extra three because the host - interleaves further work (docs, derived cells, measure sets) that consumes them. - - HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`, - `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`). - - Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer, - byte-identical). - - ⭐ WAVE 19 / R9 — `with_cohorts` NO LONGER MEANS "customer only". Wave 16 set it False on the - product surface because cohorts were a single customer-keyed bucket, so resolving them against - product pids would have intersected two unrelated id spaces and printed a plausible, - meaningless member count. `modules.cohort` is scope-parameterized now: the lists come from - THIS topic's bucket (`cohort_mod.scoped(scope_key)`), so their ids are this topic's ids and - the intersection with `pool_pids` is the ordinary one. The flag survives as an honest OFF - switch for a surface that wants no membership channel at all — it is not a topic wall. - """ - import modules.cohort as cohort_mod - - cohort_lists = [] - if with_cohorts: - for cid, c in sorted(cohort_mod.scoped(scope_key).visible(uname, pool_pids).items(), - key=lambda kv: (kv[1].get('name') or '').lower()): - members = [p for p in (c.get('members') or []) if p in pool_pids] - entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members} - # Rule 8b: a member can drop out of the 24-month pool without the cohort being - # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule - # forbids. - missing = len(c.get('members') or []) - len(members) - if missing: - entry['missing'] = missing - cohort_lists.append(entry) - - fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key, - fields_base=fields_base) - views = views_from_defs(defs or {}, ws.get('views'), fields, - # Wave 21 (item 3, R6): the system view's name is TOPIC-DERIVED. A - # user database's default view used to read "All customers" — a - # compiled customer literal minted on every topic, one half of the - # owner's "my new database looks like RI's Customer table". The ID - # stays 'all-customers' everywhere (pinned client+server — the - # client's UNDELETABLE set and the view pin both name it). - system_name=("All products" if scope_key == 'product' - else "All records" - if str(scope_key or '').startswith('ut_') - else "All customers"), - locked_lists=cohort_lists, - # ⭐ WAVE-27 item 5 (C7) — this user's own rail arrangement, from - # their own stratum. Read here rather than sorted by the client so - # the ORDER a request answers with is the order that was stored: - # sorting client-side would make the rail settle after a paint on - # every load, and shared views would land in server order first. - view_order=ws.get('viewOrder')) - workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists} - # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy - # wins when present; this is the server's answer for a FRESH browser, which used to fall - # all the way to the system default view (and whatever display mode was stored on it). - if ws.get('activeViewId'): - workspace['activeViewId'] = str(ws['activeViewId']) - - # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be - # deleted by a path that knows nothing about folders, and the placement map must not outlive - # the thing it points at. - _folders = clean_folders(ws.get('folders')) - _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')} - # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT - # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so - # its folder has to be an ordinary view folder — but rewriting the stored map would be a - # one-shot write that has to be got right once, while this is a projection that is right - # every time it runs. New drags write to `views` anyway (the client only knows that - # surface), so `cohorts` drains on its own and never needs a second pass. - # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged: - # re-parenting somebody's list into a folder that merely shares an id is a worse outcome - # than the list appearing at the root, where it is visible and one drag from home. - _cf = list(_folders.get('cohorts') or []) - if _cf: - _vf = list(_folders.get('views') or []) - _taken = {f['id'] for f in _vf} - _order = len(_vf) - for _f in _cf: - if _f['id'] in _taken: - continue - _vf.append({**_f, 'order': _order}) - _order += 1 - _folders['views'] = _vf - _raw_item_folders = dict(ws.get('itemFolders') or {}) - if _raw_item_folders.get('cohorts'): - # Cohort placements now describe VIEWS (same ids — that is the point of preserving them). - # A placement already stored on the views surface WINS: it is the more recent act. - _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}), - **dict(_raw_item_folders.get('views') or {})} - _placed = clean_item_folders( - _raw_item_folders, _folders, - {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}}) - if _folders.get('views'): - workspace['folders'] = _folders['views'] - # ⛔ `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a - # wire that still described one would invite a second rendering of rows that are now views. - _vplaced = _placed.get('views') or {} - for _v in (views or []): - if isinstance(_v, dict) and _v.get('id') in _vplaced: - _v['folderId'] = _vplaced[_v['id']] - _cplaced = _placed.get('cohorts') or {} - for _c in cohort_lists: - if _c['id'] in _cplaced: - _c['folderId'] = _cplaced[_c['id']] - - # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order, - # re-validated at SERVE time exactly like folders — a field can be deleted by a path - # that knows nothing about this stratum, and a stale key must not outlive its field. - _rl = ws.get('recordLayout') - if isinstance(_rl, dict) and isinstance(_rl.get('order'), list): - _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')} - _order, _seen = [], set() - for _k in _rl['order'][:200]: - _k = str(_k or '') - if _k and _k in _fkeys and _k not in _seen: - _seen.add(_k) - _order.append(_k) - if _order: - workspace['recordLayout'] = {'order': _order} - - return workspace, fields, views, cohort_lists - - -def embed_html_path(): - """The first existing candidate path for the inlined single-file build, or None.""" - for p in _EMBED_CANDIDATES: - if p.is_file(): - return p - return None - - -def scope_counts(shown, matched, total): - """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). - - `matched` and `total` MUST come from their own queries over the whole scope. Never pass - `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would - report the window size as though it were the result size. - - Refuses the shapes that could only be a mistake, because a wrong count here is invisible on - screen (it looks like a smaller dataset, not like an error). - """ - shown, matched, total = int(shown), int(matched), int(total) - if matched > total: - raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " - f"more rows than the scope holds") - if shown > matched: - raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " - f"more rows than the filter matched") - return {"shown": shown, "matched": matched, "total": total, "windowed": True} - - - -# ⛔ EXIT-6 (2026-08-04): `build_html`, `component_dir`, `render` and `_DECLARED_COMPONENTS` WERE -# HERE, and they are gone with Streamlit. They were the EMBED HOST — the path that declared the -# prebuilt bundle as a `streamlit.components.v1` custom component (or injected the single-file -# HTML build as a fallback) so the React grid could be drawn inside a Streamlit page. -# -# THIS MODULE ITSELF SURVIVES, and that distinction is the whole point: `aios_grid.py` is imported -# at 11 sites across `aios-web/api/` plus `harness/semantic.py` — it owns the canonical field -# contract, the workspace wire and the count envelope. Only the ~95 lines that knew about a HOST -# went; the rest never did. Its one and only `import streamlit` lived inside `render`, lazily, and -# left with it. `api/verify_no_streamlit.py` now gates that nothing here re-imports it. -# -# Deleted with them: `aios_grid_embed.html` + `aios_grid_component/index.html` (a 2.1 MB prebuilt -# bundle), `build_embed.py` that produced them, and `deploy_hf.py`'s embed-staleness guard. The -# React app is now served directly by the FastAPI container — there is no twin to keep fresh, so -# the entire class of "the code shipped but the bundle did not" is retired rather than guarded. +"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. + +This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the +standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit +component. The React bundle reads its data from `window.__AIOS_GRID__` (an object +`{fields, rows}`) that we inject into the page before the app's module script runs +— so there is NO backend and NO /api call in the Streamlit container; the browser only ever +sees the derived JSON we hand it. + +Usage (from any page): + import aios_grid + aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) + +Design notes: + * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and + `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and + sidesteps deploy_hf.py's import guard entirely. + * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to + deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy + dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py). + For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. + * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args + into the React app and returns guarded events for persistence in the tenant store. The old + injected-HTML path remains a read/local-write fallback when component assets are absent. +""" +import json +import re +from pathlib import Path + +_HERE = Path(__file__).resolve().parent + +# Where the inlined single-file build lives. FIRST match wins: +# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) +# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) +_EMBED_CANDIDATES = [ + _HERE / "aios_grid_embed.html", + _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", +] +_COMPONENT_CANDIDATES = [ + _HERE / "aios_grid_component", + _HERE.parent / "aios-web" / "web" / "dist-embed", +] +_DECLARED_COMPONENTS = {} + +# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS +# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is +# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. +# +# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so +# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the +# module preserves that exactly — no import, no guard interaction — while still single-sourcing +# the values (a build-time copy from another tree would reintroduce the drift we're removing). +# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run +# aios-web/verify_fields_contract.py. --- +_FIELDS_PATH = _HERE / "aios_grid_fields.json" + + +def _load_fields(): + if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " + "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." + ) + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = doc.get("fields") if isinstance(doc, dict) else doc + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") + return fields + + +FIELDS = _load_fields() + + +def product_fields(): + """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same + JSON). A separate accessor rather than a second module constant so the one file-read and the + one failure mode stay shared with `FIELDS`.""" + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = (doc.get("product_data") or {}).get("fields") + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.") + return fields + +# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — +# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. + + +def _round(v): + return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + +# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors +# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. +# select — a single-select with its own `options` (the "Status" a user wants to add; distinct +# from the Odoo-sourced `status` lifecycle, which is not user-defined) +# user — an assignee, whose choices come from the HOST's real user list, never from here +#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared +#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every +#: member of it (the `multi` grouping/cell contract the Cohorts column established). +#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · +#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level +#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's +#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). +#: ⭐ Wave-19 R7 / contract C5: `image` — a PICTURE on a record. The cell holds a string +#: REFERENCE, never bytes: a product `code`, an `ed:` editorial asset, or `rec:` for +#: something uploaded through the field (`POST /api/v1/assets/records`). Storing a reference is +#: what lets Royal's 1,142 existing SKU masters appear with nothing re-uploaded, and it keeps the +#: overlay stratum the size it is — base64 bytes in a JSON blob read on every render would be a +#: megabyte-per-row tax on the whole store. Editable by NATURE (deliberately NOT in +#: `READONLY_CUSTOM_TYPES`): the ref is what the upload endpoint hands back, and the client PATCHes +#: it through the ordinary overlay wall rather than the asset route writing cells behind it. +#: ⭐ WAVE 23 (C7) — `json` joined: a cell holding a whole DOCUMENT (an Instagram comment thread, +#: a webhook payload, a scraped blob) that opens in its own viewer instead of being flattened +#: into one unreadable line. It is EDITABLE by nature, like `image`: the value is still a plain +#: string on the wire, so it rides the ordinary overlay wall — what makes it a json field is that +#: `grid_events` REFUSES a write that does not parse (a column promising structure must not +#: silently hold something that isn't). +#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They ride here because +#: `UT_FIELD_TYPES` must stay a SUBSET of this set (gated in verify_api's W18-UT section) — the +#: surfaces that CREATE them are the user-table databases, where a relation between two tables is +#: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to +#: point at, so the column menu there simply never offers one. +#: ⭐⭐ WAVE-34 (owner ruling R13) — `ai_enrich` JOINS, and it had to join HERE in the same change +#: that put it in `UT_FIELD_TYPES`, not a ticket later. The wave planned a SERVER-FIRST landing on +#: the reasoning that a kind the client does not offer is invisible while a kind the server refuses +#: deletes a column. That reasoning is sound and the conclusion was still wrong, because THREE +#: parity gates chain over these sets and none of them permits a partial landing: +#: `verify_api` W18-UT `UT_FIELD_TYPES - CUSTOM_FIELD_TYPES == set()` (this line) +#: `aw_fields_contract` §5 `CREATABLE_TYPES == CUSTOM_FIELD_TYPES`, EXACT set equality +#: `types.ts` `CREATABLE_TYPES: readonly FieldType[]`, so the union must carry it too +#: Measured live by lane B at 17:09: `api_api` went 969/969 to 968/969 the moment the kind entered +#: `core/user_tables.py` alone, printing `got {'ai_enrich'} want set()`. The comments on +#: `json`, `link`/`rollup` and `code` below all say the same thing in their own words. +CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", + "checkbox", "phone", "email", "url", "rating", "created_time", "formula", + "automation", "image", "json", "link", "rollup", "code", "ai_enrich"} + +#: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. +#: +#: R13 is explicit that this kind is "syntax-highlighted storage + language config, NO execution +#: engine". So a language is a RENDERING hint and nothing else: it selects a highlighter, it never +#: selects an interpreter, and no value here may ever grow a run path. The list is short on +#: purpose — every entry costs a highlighter the client actually has to implement, and an +#: unimplemented language would paint plain text under a label promising colour. +#: +#: `plain` is the default and the fallback, so it is never a second way to say nothing: it is the +#: honest answer for a snippet whose language the user has not chosen. +CODE_LANGUAGES = {"plain", "json", "sql", "python", "javascript", "typescript", + "html", "css", "markdown", "yaml", "xml", "shell"} + + +def _clean_code(raw): + """The `code` field's config bag -> `{'language': ...}`, or None. + + Deliberately OPTIONAL rather than required (the `automation` posture, not `link`'s): a code + column with no declared language is a legitimate state — it stores and highlights as plain + text — so refusing the FIELD over a missing bag would block the ordinary create path. An + unknown language falls back to `plain` rather than refusing, because the value is a rendering + hint: dropping the user's column to punish a typo in a highlighter name would be the + disproportionate half of the fail-closed rule. + + ⚠ `plain` RETURNS NONE, and that is what makes the control reversible rather than one-way. + Absent already means plain, so storing `{'language': 'plain'}` would be the default wearing a + second name (the `kanbanClamp` law). But the patch path resolves an OMITTED key to the + previous value — so if plain were merely omitted by the client, switching a column back from + SQL to Plain text would keep storing SQL and read as a control that does not save. Sending + the bag explicitly and having it evaluate to None here means: omit = keep, plain = clear. + """ + if not isinstance(raw, dict): + return None + lang = str(raw.get('language') or 'plain').strip().lower() + if lang not in CODE_LANGUAGES or lang == 'plain': + return None + return {'language': lang} +#: User-created types whose CELLS are read-only: their values are computed (formula — client +#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). +#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so +#: the client never offers an editor and the host never accepts a cell write for them. +READONLY_CUSTOM_TYPES = {"created_time", "formula"} +MAX_FIELD_OPTIONS = 50 +MAX_FORMULA_LEN = 500 +#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. +RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 + +#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that +#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's +#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata +#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a +#: measure field is READ-ONLY and its values are computed by the host per render. +MEASURE_FIELD_PREFIX = "measure_" +#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). +MEASURE_FIELD_TYPES = {"currency", "int", "pct"} + + +def _clean_options(raw): + """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. + Mirrors types.ts parseOptions — a choice list that means one thing in the picker and + another in the filter dropdown is a column with two vocabularies.""" + out, seen = [], set() + for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: + if not isinstance(v, (str, int, float)) or isinstance(v, bool): + continue + s = str(v).strip()[:120] + if not s or s.lower() in seen: + continue + seen.add(s.lower()) + out.append(s) + return out[:MAX_FIELD_OPTIONS] + + +def _clean_option_colors(raw, options): + """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary.""" + if not isinstance(raw, dict): + return {} + supplied = {} + for label, color in raw.items(): + if not isinstance(label, str) or not isinstance(color, str): + continue + clean = color.strip().upper() + if re.fullmatch(r"#[0-9A-F]{6}", clean): + supplied[label.strip().lower()] = clean + out = {} + for option in options or []: + color = supplied.get(str(option).strip().lower()) + if color: + out[str(option)] = color + return out + + +def _choice_appearance(raw, options): + """Validated select-family appearance. Absent colour toggle means legacy-on.""" + if not isinstance(raw, dict): + return {} + out = {} + if isinstance(raw.get("colorCodeOptions"), bool): + out["colorCodeOptions"] = raw["colorCodeOptions"] + colors = _clean_option_colors(raw.get("optionColors"), options) + if colors: + out["optionColors"] = colors + return out + + +def _clean_rating_max(raw): + """A rating's star count, bounded. Anything unusable is the default, not a refusal — the + field still holds its 1..max integers either way.""" + try: + return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) + except (TypeError, ValueError): + return RATING_MAX_DEFAULT + + +#: The field types a number-style display format may apply to. `formula` is here because its +#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. +_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} + + +def _clean_format(raw, ftype): + """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong + types return None (the property is simply absent). Rendering-only — a format can change how + a value reads, never what it is, which is why this needs no parity gate of its own.""" + if not isinstance(raw, dict): + return None + out = {} + if ftype in _NUMBER_FORMAT_TYPES: + if isinstance(raw.get("thousands"), bool): + out["thousands"] = raw["thousands"] + if raw.get("decimals") is not None: + try: + d = int(raw["decimals"]) + except (TypeError, ValueError): + d = None + if d is not None and 0 <= d <= 4: + out["decimals"] = d + if isinstance(raw.get("abbrev"), bool): + out["abbrev"] = raw["abbrev"] + elif ftype in ("date", "created_time"): + if isinstance(raw.get("time"), bool): + out["time"] = raw["time"] + if raw.get("tz") in ("local", "utc"): + out["tz"] = raw["tz"] + return out or None + + +def _clean_permissions(raw): + """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is + the host handler's business (creator/admin, enforced fail-closed there); this validates + only the shape, like every other property here.""" + if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): + return {"edit": raw["edit"]} + return None + + +#: ⛔ THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is +#: refused by a wall that is supposed to be structural (2026-08-03). +#: +#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client +#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` — CONCATENATE, TEXT, +#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so +#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace` +#: dropped the column on read. Nothing went red — a refused create looks like a quiet failure +#: and a dropped column looks like a column nobody made. +#: +#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now", +#: "OK")` and could not be created through the product UI at all. +#: +#: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar — that is the filter_sql-class +#: drift risk this file's docstring names. The charset says which characters may appear; the +#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote +#: character brings with it. +_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") +_FORMULA_REF = re.compile(r"\{([^{}]*)\}") + + +def _quotes_balanced(s): + """An even number of `"` — the structural half of string support. + + Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it + contributes TWO characters. So a well-formed expression always has an even count and an + unterminated string always has an odd one. What a balanced pair MEANS is the engine's + business, exactly as with parentheses. + """ + return s.count('"') % 2 == 0 + + +#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that +#: silently never runs; `source` names where the run's subject comes from. +AUTOMATION_KINDS = {"instagram_profile"} +AUTOMATION_SOURCES = {"record_url_field", "self"} +MAX_AUTOMATION_SETTINGS = 12 + + +def _clean_automation(raw, valid_keys=None, flow_ids=None): + """Validate an `automation` config bag. Returns the clean dict, or None when there is + nothing valid to store (the column then renders as unconfigured — never invented). + + ⛔ WAVE 22 (contract C8, owner item 5) — NO FIELD WITHOUT A FLOW. With `flow_ids` given + (the WRITE path: grid_events / user_tables pass the tenant's automation-definition ids), + the bag MUST carry a `flowId` naming one of them — absent or naming a deleted flow is + refused, the same fail-closed direction as a formula ref that names no field. With + `flow_ids=None` (the READ path, `fields_from_workspace`) the law is NOT applied: a column + stored before the law must keep projecting — enforcement at read time would vaporise it, + which is the `_clean_formula` write/read split exactly. + """ + if not isinstance(raw, dict): + return None + kind = str(raw.get('kind') or '').strip() + if kind not in AUTOMATION_KINDS: + return None + source = str(raw.get('source') or 'record_url_field').strip() + if source not in AUTOMATION_SOURCES: + source = 'record_url_field' + out = {'kind': kind, 'source': source} + flow = str(raw.get('flowId') or '').strip()[:40] + if flow_ids is not None and (not flow or flow not in flow_ids): + return None + if flow: + out['flowId'] = flow + url_field = str(raw.get('urlField') or '').strip()[:80] + # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time + if url_field and (valid_keys is None or url_field in valid_keys): + out['urlField'] = url_field + settings = {} + for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]: + if isinstance(v, bool) or isinstance(v, (int, float)): + settings[str(k)[:40]] = v + elif isinstance(v, str): + settings[str(k)[:40]] = v[:200] + if settings: + out['settings'] = settings + return out + + +def _clean_formula(raw, valid_keys=None): + """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). + + Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` + refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a + wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a + Python mirror of the grammar: a second engine is the filter_sql-class drift risk. + Structure IS checked — charset, length, balanced parens, BALANCED QUOTES, well-formed + non-empty `{refs}` — and at WRITE time (`valid_keys` given) every ref must name a field this + table has, fail closed. At READ time refs are left alone: a referenced field deleted later + must blank the CELLS, not vaporise the column. + + ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so + `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the + fail-closed direction: the alternative is teaching this function where strings begin and + end, which is the second grammar the paragraph above refuses to write. + """ + if not isinstance(raw, str): + return None + s = raw.strip() + if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): + return None + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return None + if depth: + return None + if not _quotes_balanced(s): + return None + refs = _FORMULA_REF.findall(s) + leftover = _FORMULA_REF.sub("", s) + if "{" in leftover or "}" in leftover: # unbalanced / nested braces + return None + if any(not r.strip() for r in refs): # a `{}` ref names nothing + return None + if valid_keys is not None and any(r not in valid_keys for r in refs): + return None + return s + + +def _field_extras(saved, ftype): + """createdBy / permissions / format / scope — the validated passthrough the created strata + share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it + is carried so the client can gate its menus and the handler can enforce against it. + `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler + stamps it at create (cohort page only) and preserves it like createdBy; carried here so the + client can label the field, filtered OUT of other pages by fields_from_workspace.""" + out = {} + who = saved.get("createdBy") + if isinstance(who, str) and who.strip(): + out["createdBy"] = who.strip()[:80] + perms = _clean_permissions(saved.get("permissions")) + if perms: + out["permissions"] = perms + fmt = _clean_format(saved.get("format"), ftype) + if fmt: + out["format"] = fmt + if saved.get("scope") == "cohort": + out["scope"] = "cohort" + corrected_from = saved.get("labelCorrectedFrom") + correction_id = saved.get("labelCorrectionId") + if (isinstance(corrected_from, str) and corrected_from.strip() + and isinstance(correction_id, str) and correction_id.strip()): + out["labelCorrectedFrom"] = corrected_from.strip()[:120] + out["labelCorrectionId"] = correction_id.strip()[:180] + return out + + +#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). +#: +#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the +#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the +#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. +#: Putting it in the canonical contract would mean an always-present column that is empty for +#: everyone else, plus three consumers to keep in step for a value none of them can produce. +COHORT_COLUMN = 'cohorts' + + +def cohort_field(label='Locked views'): + """The derived membership column's descriptor. + + ⚠ WAVE 17 item 14 / C-STR — THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES + NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because + every stored view that shows or groups by this column names it by KEY, and every gate in + two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it + Cohort, but locked instead"), which is a change to what a reader sees — renaming the + identifiers would break saved views to change a word nobody reads. + + `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability + off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay + keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what + stops the column menu calling it a "source field" on that basis. + + ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another + column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains + "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse + than one that is absent. + """ + return { + 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', + 'default': False, 'filterable': False, 'derived': True, 'multi': True, + 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' + 'so grouping by this column puts a customer under EVERY view they are in — the ' + 'group counts therefore add up to more than the record count, which stays the ' + 'number of distinct customers. Read-only: membership changes only by adding or ' + 'removing customers on the locked view itself.', + } + + +def cohort_cells(cohorts, allowed_pids=None): + """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. + + Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, + never merged into the cached pool rows — those are shared across users, and stamping one + user's cohorts onto them would leak the membership to everybody else on the next render. + """ + cells = {} + for c in cohorts or []: + # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY + # cohort they are in, so it cannot also occur inside a name. Cohort names are free text + # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather + # than left to break the split silently — one group called "Q3 calls" and another called + # "west" would be two lists that do not exist. The cost is cosmetic and confined to the + # cell; the Cohort page still shows the name the user typed. + name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() + for pid in c.get('pids') or (): + if allowed_pids is not None and pid not in allowed_pids: + continue + cells.setdefault(pid, []).append(name) + return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} + + +def clean_measure_field(raw, offered): + """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. + + `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — + the same admission the condition builder uses, so a field can only name a measure this + caller could also filter by. Returns the canonical stored shape, or None (fail closed). + + `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: + the client keys editability off `source == 'overlay'` and the host accepts cell writes only + for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` + because the REPLACEMENT is the measure CONDITION with the same measure and window — the + governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. + """ + if not isinstance(raw, dict): + return None + key = str(raw.get("key") or "") + if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: + return None + spec = raw.get("measure") + if not isinstance(spec, dict): + return None + m = (offered or {}).get(spec.get("key")) + if not m: + return None # not admitted for this caller -> fail closed + window = _clean_window(spec.get("window")) + if window is None: + return None # a measure column with no period is not a column + mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" + return { + "key": key, + "label": str(raw.get("label") or m.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": True, + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(raw.get("note") or "")[:2000], + "measure": {"key": str(spec.get("key"))[:80], "window": window}, + } + + +def measure_fields_of(fields): + """The formula-measure columns among `fields` — the ones whose values the host must compute + per render (see `rows_from_pool`'s `derived`).""" + return [f for f in fields or [] if isinstance(f.get("measure"), dict)] + + +def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None): + """Overlay persisted notes/custom fields onto the immutable source-field contract. + + `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and + `default: False`, so it never displaces a column somebody already reads; the Fields menu is + where you turn it on. + + Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), + and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure + field was validated against the caller's measure OFFER when it was written + (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero + app-internal imports and cannot know the offer. A measure that has since become + unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value + time, never to an error. + + `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A + saved def carrying `scope` is emitted only when it matches — so a cohort-specific field + never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees + only unscoped (global) fields; base contract fields are never scoped. + + `fields_base` (wave 16 C-TOPIC) — the canonical contract to overlay onto. Absent = the + CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product + surface passes `product_fields()`. The workspace dict a caller hands in must already be the + matching table object's bucket — this function cannot tell a customer overlay from a + product one, which is exactly why the buckets are separate stores. + """ + saved = dict((workspace or {}).get("fields") or {}) + out = [] + base_fields = fields_base if fields_base is not None else FIELDS + base_keys = {field["key"] for field in base_fields} + for base in base_fields: + meta = saved.get(base["key"]) or {} + field = dict(base) + if isinstance(meta.get("note"), str): + field["note"] = meta["note"][:2000] + # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a + # number or date READS, per user. Rendering-only, so this is the whole acceptance. + fmt = _clean_format(meta.get("format"), base.get("type")) + if fmt: + field["format"] = fmt + # ⭐ W29-T83 — the saved COLUMN SUMMARY, the read half of the write door in + # `grid_events.field_upsert`. Without this the value round-trips into the store and is + # never served back, which looks exactly like a write that never happened + # ([[read-path-cannot-witness-write-path]]). Absent = whatever the contract declares. + if meta.get("agg") in FIELD_AGGS: + field["agg"] = meta["agg"] + # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved + # window/label override. Wave 6 deleted every preset member (the owner's + # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the + # measure_ path's twin for any future preset-carrying contract, and because deleting + # it would silently change what a re-added preset means. + if base.get("preset") and isinstance(base.get("measure"), dict): + saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} + window = _clean_window(saved_measure.get("window")) + if window is not None: + field["measure"] = {"key": base["measure"]["key"], "window": window} + if isinstance(meta.get("label"), str) and meta["label"].strip(): + field["label"] = meta["label"][:120] + out.append(field) + for key, field in saved.items(): + if key in base_keys or not isinstance(field, dict): + continue + if field.get("scope") and field.get("scope") != scope_key: + continue # a cohort-specific field on another page (wave-6 item 9) + if (str(key).startswith(MEASURE_FIELD_PREFIX) + and isinstance(field.get("measure"), dict)): + window = _clean_window(field["measure"].get("window")) + mkey = str(field["measure"].get("key") or "") + if window is None or not mkey: + continue + mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": bool(field.get("default", True)), + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(field.get("note") or "")[:2000], + "measure": {"key": mkey[:80], "window": window}, + **_field_extras(field, mtype), + }) + continue + if not str(key).startswith("custom_"): + continue + ftype = field.get("type") + if ftype not in CUSTOM_FIELD_TYPES: + continue + if ftype in READONLY_CUSTOM_TYPES: + # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort + # column's mechanism (source 'odoo' + derived) so the client never offers an + # editor and the host's overlay-write guard excludes them by construction. + # FILTERABLE since wave 6 (owner item 6): their values live client-side + # (formula computes over the row, created_time renders `_created`), this + # table's counts are client-mode, and the windowed count path never sees + # these tables — so the client engine answers them soundly. + entry = { + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "odoo", + "derived": True, + "filterable": True, + "default": bool(field.get("default", True)), + "custom": True, + "note": str(field.get("note") or "")[:2000], + **_field_extras(field, ftype), + } + if ftype == "formula": + formula = _clean_formula(field.get("formula")) + if formula is None: + continue # a formula field without a formula is nothing + entry["formula"] = formula + out.append(entry) + continue + if field.get("source") != "overlay": + continue + options = (_clean_options(field.get("options")) + if ftype in ("select", "multiselect") else []) + if ftype in ("select", "multiselect") and not options: + # A select with no surviving choices can never hold a value. Dropping the COLUMN + # would lose the user's data; degrading it to text keeps every stored value + # readable and lets them re-add choices. + ftype = "text" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "overlay", + "default": bool(field.get("default", True)), + "custom": True, + # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a + # picker offering Average against a projection that only passes Sum through is the + # silent half of this feature. + "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, + "note": str(field.get("note") or "")[:2000], + **({"options": options} if ftype in ("select", "multiselect") else {}), + **(_choice_appearance(field, options) + if ftype in ("select", "multiselect") else {}), + # comma-joined SET semantics (the Cohorts column's contract): the row belongs to + # every member, groups count it under each, the toolbar count stays distinct. + **({"multi": True} if ftype == "multiselect" else {}), + **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), + **({"automation": _clean_automation(field.get("automation"))} + if ftype == "automation" and _clean_automation(field.get("automation")) else {}), + # ⭐ WAVE-27 item 13 (R13) — the code column's language rides the wire, because the + # highlighter is chosen per column and the client cannot infer a language from a + # string. Absent = `plain`, which is what an unconfigured code column renders as. + **({"code": _clean_code(field.get("code"))} + if ftype == "code" and _clean_code(field.get("code")) else {}), + **_field_extras(field, ftype), + }) + if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): + # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. + # A user table's field keys are slugged from whatever its creator typed, so a column + # literally called "Cohorts" produces the key `cohorts` — and appending here unguarded + # would put TWO fields with one key on the wire. The client indexes fields by key, so the + # duplicate does not error: it silently paints one column's values under the other's + # header. The user's own column wins; the derived one steps aside rather than shadowing it. + out.append(cohort_field()) + return out + + +def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): + """Map customer_data.pool() dicts -> the API row shape the grid expects: + pid + each Odoo field (numeric fields rounded, text/date passed through) + the + persisted external overlay. Mirrors the standalone API payload contract. + + `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than + reads off the pool row — today just the cohort column. A separate argument from `overlays` + on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is + never written back would make the dict mean two things. + """ + fields = fields or FIELDS + odoo_fields = [field for field in fields if field["source"] == "odoo"] + overlay_fields = [field for field in fields if field["source"] == "overlay"] + derived_keys = [field["key"] for field in fields if field.get("derived")] + overlays = overlays or {} + derived = derived or {} + out = [] + for r in pool_rows: + pid = r.get("pid") + # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the + # `created_time` field type renders, regardless of that field's own key. Not a Field: + # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride + # the same way: the Map VIEW's data, nullable, deliberately not a column. + row = {"pid": pid, "_created": r.get("_created") or "", + "lat": r.get("lat"), "lon": r.get("lon")} + for field in odoo_fields: + k = field["key"] + if field.get("derived"): + continue # not on the pool row — filled from `derived` below + v = r.get(k) + row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) + saved = overlays.get(str(pid), {}) or {} + for field in overlay_fields: + row[field["key"]] = saved.get(field["key"], "") + got = derived.get(pid) or {} + for k in derived_keys: + # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a + # text column is the question somebody will ask of it. + row[k] = got.get(k, "") + out.append(row) + return out + + +# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- +# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are +# VALUE-FREE (they legitimately carry no value and must never be dropped for it). +FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', + 'gt', 'gte', 'lt', 'lte', 'between', 'within', + # Wave 2026-08-02 (C-OPS): RANK operators — evaluated as a SET pass over the + # sibling-filtered domain by the client engine (useVisibleRows). The validator + # accepts them like any op (structural, not semantic); filter_sql REFUSES to + # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg / + # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a + # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100, + # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a + # measure-carrying column they rank the field's own derived values. + 'topN', 'bottomN', 'inTopPct', 'inBottomPct', + 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'} +#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not +#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean +#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. +COHORT_FIELD = '__cohort__' +#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. +#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: +#: a set op reaching a column leaf would fall through the client engine's switch to "no +#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. +COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} +#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: +#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the +#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. +COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} +#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. +MAX_COHORT_IDS = 20 + + +def parse_cohort_ids(value): + """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. + + Comma-separated in one string because `FilterRule.value` is what all four layers persist and + round-trip, and a one-element list is byte-identical to what the single-cohort leaf already + stored — so every shipped view parses with no migration. Safe because a cohort id is built + from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. + """ + out = [] + for raw in ('' if value is None else str(value)).split(','): + cid = raw.strip()[:120] + if not cid or cid in out: + continue + out.append(cid) + if len(out) >= MAX_COHORT_IDS: + break + return out +# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays +# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. +MAX_FILTER_DEPTH = 3 +MAX_FILTER_NODES = 100 # total nodes across the whole tree +MAX_FILTER_SIBLINGS = 50 # per level + + +#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by +#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored +#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already +#: need a gate to prevent. This validates SHAPE only. +WINDOW_MAX_N = 3650 + + +def _clean_window(raw): + """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. + + Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by + `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this + way keeps the grid module reusable and keeps one definition of what "last quarter" means. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if not isinstance(kind, str) or not kind or len(kind) > 40: + return None + out = {'kind': kind} + if raw.get('n') is not None: + try: + out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) + except (TypeError, ValueError): + return None + for side in ('from', 'to'): + if raw.get(side) not in (None, ''): + out[side] = str(raw[side])[:32] + return out + + +def _clean_rhs(raw, valid_keys): + """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. + + SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` + with the measure keys, exactly as it does for the left side), and a measure rhs must carry a + window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in ('field', 'measure', 'stat'): + return None + if kind == 'stat': + # A STATISTIC carries no column: the population is the comparand. Shape only — which + # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one + # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. + stat = raw.get('stat') + if not isinstance(stat, str) or not stat or len(stat) > 24: + return None + return {'kind': 'stat', 'stat': stat} + col = raw.get('colId') + if col not in valid_keys: + return None + out = {'kind': kind, 'colId': col} + if kind == 'measure': + window = _clean_window(raw.get('window')) + if window is None: + return None # a measure comparand with no period is not a question + out['window'] = window + return out + + +#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 +#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown +#: display means, so it is never stored. +#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to +#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the +#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing +#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live +#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The +#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on +#: READ forever, only ever WRITE 'chart'. +#: ⭐ WAVE-27 item 8 (owner ruling R2, contract C3): 'swipe' — a DECK of the records whose bound +#: single-select is EMPTY, triaged one at a time by swiping left or right into two of that +#: field's options. Landed here FIRST and in the same change as the client registry, which is +#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an +#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view +#: that silently reverts to a grid on the next read. +#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has +#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never +#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — +#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door +#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at +#: anything. The mirror is one name, and it is the half nobody could see was missing because +#: BOTH sides were individually consistent. +#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out +#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law +#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by +#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered +#: one is red too, so the hold cannot outlive its reason the way wave 27's did). +DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', + 'timeseries', 'catalog', 'swipe', 'form'} + +#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at +#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read +#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a +#: legal mode and this function had no branch for the key, so every spec a client sent was dropped +#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. +#: +#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a +#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — +#: so one tenant setting its token to another tenant's value would silently receive that tenant's +#: submissions. The token is minted server-side and lives in a bucket no client write can reach; +#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. +FORM_ACCESS = ('public', 'emails') +MAX_FORM_FIELDS = 60 +MAX_FORM_EMAILS = 200 +MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 +MAX_FORM_EMAIL = 254 + +#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the +#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing +#: from here is a question the builder cannot ask yet; a type wrongly present is a public door +#: writing values the column cannot mean (an `image` with no upload channel, a `json` document +#: typed into a text box, a `link` naming a record id a stranger guessed). +#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: +#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an +#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks +#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather +#: than re-derived ([[one-evaluator-per-question]]). +#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares +#: the two files name-for-name. +FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', + 'checkbox', 'phone', 'email', 'url', 'rating') + +#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in +#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the +#: expensive failure, not an odd address that gets in. +_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') + + +def _clean_form(raw, valid_keys): + """One form spec, fail-closed. Returns None when nothing is configured. + + ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the + builder let somebody arrange these questions, and sorting them by column order would silently + rearrange a live form every time a column was added (`_public_form` states the same rule from + the serving end). + + ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A + swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck + that can never write. A form is a LIST of questions — losing the column behind question three + costs the asker question three, and taking the whole form away because one field was deleted + would be a far larger loss than the one that happened. + """ + if not isinstance(raw, dict): + return None + fields, seen = [], set() + for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: + if k in valid_keys and k not in seen: + seen.add(k) + fields.append(k) + out = {} + if fields: + out['fields'] = fields + # A required flag on a question the form no longer asks is not a rule, it is a trap: the + # submitter can never satisfy it and the sentence names a field they cannot see. + req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] + if req: + out['required'] = req + for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), + ('submitLabel', MAX_FORM_SUBMIT)): + text = str(raw.get(key) or '').strip()[:cap] + if text: + out[key] = text + # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only + # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which + # is why `emails` is what gets written rather than a `public: false`. + if raw.get('access') == 'emails': + out['access'] = 'emails' + emails = [] + for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: + e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] + if _FORM_EMAIL.match(e) and e not in emails: + emails.append(e) + # Kept even while `access` is public: a person toggling the door open to test it and back + # again must not lose the list of people they typed. It is never served publicly. + if emails: + out['emails'] = emails + return out or None + + +#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a +#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same +#: number rather than a second opinion about it: a longer string cannot name a real option, and +#: a SHORTER cap here would silently refuse a binding to a legal one. +MAX_SWIPE_OPTION = 120 + +#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the +#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N / +#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops, +#: so an accepted save reads back byte-identically on both engines. +TS_BUCKETS = {'week', 'month', 'quarter', 'year'} +TS_MAX_LAST_N = 120 +TS_MAX_FIELDS = 12 +MAX_CALENDAR_METRICS = 4 +_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') + +#: C6-CATALOG (wave 18) — the catalog view's vocabulary and caps. types.ts mirrors every name +#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads +#: back identically on both engines. The code budget is the order-sensitive one — see +#: `_clean_catalogs`. +MAX_CATALOGS = 12 +MAX_CATALOG_PAGES = 40 +MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER +CATALOG_PAPERS = {'letter', 'a4', 'tabloid'} +CATALOG_ORIENTATIONS = {'portrait', 'landscape'} +CATALOG_QUALITIES = {'web', 'print'} +CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'} +CATALOG_COLS = (2, 3, 4) +CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80 +CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60 +_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') + +#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts +#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES — the C-DISP byte-identical law. +TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') +TS_MAX_CUSTOM_ROWS = 12 +TS_MAX_STYLES = 200 +#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client +#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole +#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. +_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') + +#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown +#: mode is still rejected rather than accidentally aliased. +_LEGACY_MODES = {'dashboard': 'chart'} + +#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's +#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a +#: kind the client cannot draw is worse than one that does not exist yet. +#: Wave-16 C-CHARTCAP: + 'table' — the group-by aggregate table (by-rep / by-BU / +#: top-customers, the third Sales block shape). Client renderer: DashboardView's +#: GroupTableView over salesParity.tableFromSpec. +CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} +CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} + +#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is +#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is +#: the picker's order; membership tests read it as a tuple perfectly well. +#: +#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` +#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): +#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole +#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum +#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are +#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both +#: would have to be the intersection of what each can express. +#: +#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` +#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, +#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would +#: give the product two words for one operation on two menus a click apart. +#: +#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column +#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. +#: +#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. +#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, +#: and must not grow a second spelling of it. +#: +#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by +#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to +#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. +FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') +MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop +MAX_CHART_TITLE = 60 + +#: Wave-9 I11 (contract C2) — chart customisation, host-validated. +#: +#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: +#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) +#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored +#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — +#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. +CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} +CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} +MAX_AXIS_LABEL = 40 +#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. +CHART_W_RANGE = (1, 12) +CHART_H_RANGE = (120, 800) + + +def _clean_chart(raw, valid_keys): + """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. + + A chart's `y` is optional (absent = count of rows, which is what "how many customers per + state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and + silently keeping it would put an empty card on the dashboard with no way to tell why. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in CHART_KINDS: + return None + x = raw.get('x') + if x not in valid_keys: + return None # dead category ref -> the chart goes, not the board + cid = raw.get('id') + if not isinstance(cid, str) or not cid.strip(): + return None # the client owns chart ids; an unidentified card + # cannot be edited or removed, so it must not persist + out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, + 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} + if raw.get('y') in valid_keys: + out['y'] = raw['y'] + else: + # no measurable column -> the only honest aggregation left is "how many rows" + out['agg'] = 'count' + title = raw.get('title') + if isinstance(title, str) and title.strip(): + out['title'] = title.strip()[:MAX_CHART_TITLE] + + # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── + # `splitBy` is the field whose values become the SERIES. It is deliberately not called + # `colorBy`: that name already means two other things here (`config.colorBy` = row + # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. + if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: + out['splitBy'] = raw['splitBy'] + # Stacking is only a question once there are series to stack, and only for the two kinds + # that can express it. Anywhere else it is dropped rather than stored as a lie the client + # would have to re-decide. + if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: + out['stacked'] = True + if raw.get('palette') in CHART_PALETTES: + out['palette'] = raw['palette'] + + axis = raw.get('axis') + if isinstance(axis, dict): + # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: + # two y-scales on one frame can manufacture any correlation you like by rescaling, and + # the honest alternatives are two charts, small multiples, or indexing to a common base. + # Ruled explicitly in contract C2 against the "Tableau versatility" brief. + clean_axis = {} + for side in ('x', 'y'): + spec = axis.get(side) + if not isinstance(spec, dict): + continue + one = {} + lab = spec.get('label') + if isinstance(lab, str) and lab.strip(): + one['label'] = lab.strip()[:MAX_AXIS_LABEL] + if spec.get('format') in CHART_FORMATS: + one['format'] = spec['format'] + if one: + clean_axis[side] = one + if clean_axis: + out['axis'] = clean_axis + + size = raw.get('size') + if isinstance(size, dict): + one = {} + for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): + try: + one[key] = max(lo, min(hi, int(size[key]))) + except (KeyError, TypeError, ValueError): + pass # a partial size is fine: the client defaults the missing axis + if one: + out['size'] = one + + # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD — the + # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed: + # a category column has no time dimension, and a stored period on it would promise a + # trend the TS channel must refuse. `span` is meaningful only beside `bucket`. + if isinstance(out.get('y'), str) and out['y'].startswith('measure_'): + if raw.get('bucket') in TS_BUCKETS: + out['bucket'] = raw['bucket'] + sp = raw.get('span') + if isinstance(sp, dict): + n = sp.get('lastN') + if (isinstance(n, int) and not isinstance(n, bool) + and 1 <= n <= TS_MAX_LAST_N): + out['span'] = {'lastN': n} + # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something — + # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta + # line). Anything else is a stored claim the renderer would have to re-refuse. + # Mirrors the client's cleanCharts rule key for key. + if raw.get('compare') == 'prior_year' and ( + out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')): + out['compare'] = 'prior_year' + return out + + +def _clean_catalog_page(raw, budget): + """C6-CATALOG — one page of a catalog. Returns `(page | None, codes_spent)`. + + `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a + page and not across the catalog: a product legitimately appears on a gallery page and again + in its section listing, and de-duplicating globally would silently delete the second + appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of + its last pages — never a random scatter, and never a page (a page with no products is a + heading the user can still see and fix). + """ + if not isinstance(raw, dict): + return None, 0 + page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX] + kind = raw.get('kind') + if not page_id or kind not in CATALOG_PAGE_KINDS: + return None, 0 + out = {'id': page_id, 'kind': kind} + for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX), + ('imageCode', CATALOG_CODE_MAX)): + v = raw.get(key) + if isinstance(v, str) and v: + out[key] = v[:cap] + products = raw.get('products') + if isinstance(products, list) and budget > 0: + clean_p, seen_p = [], set() + for c in products: + if not isinstance(c, str) or not c: + continue + c = c[:CATALOG_CODE_MAX] + if c in seen_p: + continue + seen_p.add(c) + clean_p.append(c) + if len(clean_p) >= budget: + break + if clean_p: + out['products'] = clean_p + layout = raw.get('layout') + if isinstance(layout, dict): + clean_l = {} + cols = layout.get('cols') + if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS: + clean_l['cols'] = cols + # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by + # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all). + # So only the opt-OUT is storable for the first two and only the opt-IN for the third — + # a second spelling of a default is how a round trip starts churning. + if layout.get('showPack') is False: + clean_l['showPack'] = False + if layout.get('showColor') is False: + clean_l['showColor'] = False + if layout.get('showPrice') is True: + clean_l['showPrice'] = True + if clean_l: + out['layout'] = clean_l + return out, len(out.get('products') or ()) + + +def _clean_catalogs(raw, valid_keys): + """C6-CATALOG (wave 18) — `display.catalogs`, fail-closed. Returns a list or None. + + A catalog is a PRINT artifact, so the two structural keys that decide how it paginates + (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with + no size is not a smaller catalog, it is an unrenderable one. Everything else follows the + house rules — unknown keys dropped, per-entry drops never cost the neighbours, empty + sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and + an empty one are not two spellings of the same nothing. + + `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU / + pack / colour, and `product_data` carries no pack or colour of its own — the user binds + custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same + split `dateField`/`stackField` already run. + """ + if not isinstance(raw, list): + return None + out = [] + for c in raw: + if len(out) >= MAX_CATALOGS: + break + if not isinstance(c, dict): + continue + cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX] + name = c.get('name') + # An EMPTY name is legal (the user cleared the box and will type again) — an ABSENT one + # is a malformed record. The `tsRows` label rule, same reasoning. + if not cat_id or not isinstance(name, str): + continue + cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]} + cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter' + cat['orientation'] = (c['orientation'] + if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait') + if c.get('quality') in CATALOG_QUALITIES: + cat['quality'] = c['quality'] + brand = c.get('brand') + if isinstance(brand, dict): + clean_b = {} + for k in ('primary', 'accent'): + v = brand.get(k) + if isinstance(v, str) and _HEX6.match(v): + clean_b[k] = v + company = brand.get('company') + if isinstance(company, str) and company: + clean_b['company'] = company[:CATALOG_NAME_MAX] + # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside + # print CSS is exactly the tokens-not-values rule this contract carries. + logo = brand.get('logo') + if isinstance(logo, str) and logo: + clean_b['logo'] = logo[:CATALOG_CODE_MAX] + if clean_b: + cat['brand'] = clean_b + binds = c.get('fields') + if isinstance(binds, dict): + clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price') + if binds.get(k) in valid_keys} + if clean_bind: + cat['fields'] = clean_bind + pages, budget = [], MAX_CATALOG_CODES + raw_pages = c.get('pages') + if isinstance(raw_pages, list): + for p in raw_pages: + if len(pages) >= MAX_CATALOG_PAGES: + break + page, spent = _clean_catalog_page(p, budget) + if page is None: + continue + budget -= spent + pages.append(page) + # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog + # starts in, and dropping the key would make "new" and "corrupt" the same wire value. + cat['pages'] = pages + out.append(cat) + return out or None + + +def _clean_display(raw, valid_keys): + """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. + + `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode + must be a known non-grid mode (grid is the absent default, so storing it would be a second + way to say nothing); every field ref must name a field this table has (a ref to a deleted + field is DROPPED and the client falls back to its per-mode default); unknown keys are + dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family + stack, map a single-select to colour by and a numeric to size by — is the client's + business: it is the only layer that renders them, and a wrong-typed ref degrades to that + surface's default rather than to an error (the `_clean_window` split). + + Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and + dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — + never the whole array, because losing one column should not cost the user a dashboard they + spent time building. + + W33-T45 (contract C1, ruling R5) adds `published` + `publishAccess`. ⚠ THE HALF OF THE ROUND + TRIP THIS FUNCTION CANNOT ENFORCE: `customer-grid/types.ts::cleanDisplay` is a SECOND + normalizer, in the browser, which rebuilds the config key by key on every autosave and drops + anything it does not name. A key accepted here and unknown there dies on the next column + resize — silently, because the client then POSTs the stripped config and the host REPLACES + the stored one. "The host accepts it" is half a round trip; that file is the other half. + """ + if not isinstance(raw, dict): + return None + mode = raw.get('mode') + if mode not in DISPLAY_MODES or mode == 'grid': + return None + # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown + # mode is still rejected rather than accidentally aliased into a real one. Every already + # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. + mode = _LEGACY_MODES.get(mode, mode) + out = {'mode': mode} + for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── + # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped — the new + # standardized default — so storing True would be a second way to say nothing (the same + # rule that keeps mode:'grid' out of the store). + if raw.get('kanbanClamp') is False: + out['kanbanClamp'] = False + if raw.get('calendarMode') in ('records', 'summary'): + out['calendarMode'] = raw['calendarMode'] + metrics = raw.get('calendarMetrics') + if isinstance(metrics, list): + clean_m, seen_m = [], set() + for m in metrics[:MAX_CALENDAR_METRICS]: + # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the + # user the summary card they configured around it. + if not isinstance(m, dict): + continue + mid = str(m.get('id') or '')[:40] + if (not mid or mid in seen_m or m.get('field') not in valid_keys + or m.get('agg') not in CHART_AGGS): + continue + seen_m.add(mid) + clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']}) + if clean_m: + out['calendarMetrics'] = clean_m + if raw.get('tsBucket') in TS_BUCKETS: + out['tsBucket'] = raw['tsBucket'] + span = raw.get('tsSpan') + if isinstance(span, dict): + clean_span = {} + n = span.get('lastN') + if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N: + clean_span['lastN'] = n + else: + f, t = span.get('from'), span.get('to') + f = f if isinstance(f, str) and _ISO_DAY.match(f) else None + t = t if isinstance(t, str) and _ISO_DAY.match(t) else None + if f and t and f > t: + f, t = t, f + if f: + clean_span['from'] = f + if t: + clean_span['to'] = t + if clean_span: + out['tsSpan'] = clean_span + ts_fields = raw.get('tsFields') + if isinstance(ts_fields, list): + clean_f, seen_f = [], set() + for k in ts_fields[:TS_MAX_FIELDS]: + if k in valid_keys and k not in seen_f: + seen_f.add(k) + clean_f.append(k) + if clean_f: + out['tsFields'] = clean_f + # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ──────────────────────── + deltas = raw.get('tsDeltas') + if isinstance(deltas, list): + clean_d, seen_d = [], set() + for d in deltas: + if d in TS_DELTA_KINDS and d not in seen_d: + seen_d.add(d) + clean_d.append(d) + if clean_d: + out['tsDeltas'] = clean_d + # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the + # literal opt-IN (absent = off) — the kanbanClamp asymmetry, one per direction. + if raw.get('tsGridlines') is False: + out['tsGridlines'] = False + if raw.get('tsSparkline') is True: + out['tsSparkline'] = True + rows = raw.get('tsRows') + if isinstance(rows, list): + clean_r, seen_r = [], set() + for r in rows[:TS_MAX_CUSTOM_ROWS]: + if not isinstance(r, dict): + continue + rid = str(r.get('id') or '')[:40] + r_kind = r.get('kind') + if not rid or rid in seen_r or r_kind not in ('note', 'formula'): + continue + label = r.get('label') + if not isinstance(label, str): + continue # ABSENT label = malformed; an EMPTY one is a legal spacer row + # (GRID's dated asymmetry amendments, 2026-08-02) + one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} + if r_kind == 'formula': + expr = r.get('expr') + if (isinstance(expr, str) and expr.strip() + and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): + one['expr'] = expr + # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row + # would delete the user's label to punish their arithmetic (GRID's dated + # asymmetry amendment; the calendarMetrics per-entry-drop precedent). + seen_r.add(rid) + clean_r.append(one) + if clean_r: + out['tsRows'] = clean_r + styles = raw.get('tsStyles') + if isinstance(styles, dict): + clean_s = {} + for s_key, s_val in styles.items(): + if len(clean_s) >= TS_MAX_STYLES: + break # capped, not truncated silently: the gate names this + if not isinstance(s_key, str) or not s_key or len(s_key) > 96: + continue # key = rowId or "rowId:colKey" — the client's grammar + if not isinstance(s_val, dict): + continue + one = {} + if s_val.get('bold') is True: + one['bold'] = True + if s_val.get('line') is True: + one['line'] = True + if one: + clean_s[s_key] = one + if clean_s: + out['tsStyles'] = clean_s + charts = raw.get('charts') + if isinstance(charts, list): + clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] + # de-dupe by id: two cards sharing an id are one card as far as the client's keyed + # render is concerned, and the second would silently shadow the first + seen, uniq = set(), [] + for c in clean: + if c['id'] in seen: + continue + seen.add(c['id']) + uniq.append(c) + if uniq: + out['charts'] = uniq + # ── C6-CATALOG (wave 18) ───────────────────────────────────────────────────────────── + catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) + if catalogs: + out['catalogs'] = catalogs + # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── + # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that + # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. + # Those are LISTS of independent cards, so losing one entry costs the user one card. This is + # a single three-part BINDING: a swipe view holding a fieldKey with one option, or two + # options and no field, is not a degraded swipe view — it is a deck that can never write + # anything, rendered as though it were configured. Dropping the key entirely puts the view + # back in its honest unconfigured state, which is the one state the client has a UI for. + # + # ⚠ What this CANNOT check, deliberately, and why the client must: whether `fieldKey` names + # a SELECT, and whether the two options are still in that select's vocabulary. `valid_keys` + # is a key set, and the docstring above draws this exact line — "what each mode MEANS ... is + # the client's business". So SwipeView owns three losses this function is blind to (field + # deleted, field retyped away from select, option removed) and must SHOW each one rather + # than fall back to the first option, per the `viewModes.tsx` house rule. + swipe = raw.get('swipe') + if isinstance(swipe, dict): + f_key = swipe.get('fieldKey') + left, right = swipe.get('leftOption'), swipe.get('rightOption') + ok = (f_key in valid_keys + and isinstance(left, str) and isinstance(right, str)) + if ok: + left, right = left.strip()[:MAX_SWIPE_OPTION], right.strip()[:MAX_SWIPE_OPTION] + # Both non-empty, and DISTINCT: one option on both sides is a deck whose two + # gestures do the same thing, which is two spellings of one state (the + # `kanbanClamp` law) wearing a control that promises a choice. + if left and right and left.casefold() != right.casefold(): + out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} + # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. + form = _clean_form(raw.get('form'), valid_keys) + if form: + out['form'] = form + # ── ⭐⭐ W33-T45 / CONTRACT C1 / RULING R5 (owner item 8b): IS THIS INTERFACE PUBLISHED ──── + # + # ⛔ TWO KEYS LIVE HERE AND TWO DELIBERATELY DO NOT. The `published` flag and the sharer's + # `public | password` choice are DISPLAY state — the view says what it is, the client renders + # a badge from it, and it travels with the view like every other key in this dict. The SECRET + # TOKEN and the PASSPHRASE HASH do not: they live in the server-only bucket, exactly as + # `routes_forms.py`'s `TOKENS_KEY` holds the form token. `config.display` is echoed back to + # every user who can open the view, so a token in here is a token published to the audience + # the password was meant to exclude. `_clean_form` above carries the same rule and the same + # reason; this is the second door, not a new one. + # + # ⛔ AND THE COERCION IS FAIL-CLOSED, WHICH IS WHY THIS IS NOT A BARE ALLOWLIST. A plain + # allowlist drops an unrecognised `publishAccess` and KEEPS `published: True` — leaving a + # published view with no stated access, i.e. a third state neither the ruling nor the client + # has a meaning for, on the one key where guessing wrong publishes a tenant's data to the + # open internet. So: a published view ALWAYS carries an access, and anything that is not the + # literal `'public'` reads as `'password'`. The unpublished case stores nothing at all — + # absent means unpublished, and a `published: False` would be the second way to say nothing + # that `kanbanClamp` and `mode: 'grid'` are both here to forbid. + if raw.get('published') is True: + out['published'] = True + out['publishAccess'] = 'public' if raw.get('publishAccess') == 'public' else 'password' + return out + + +#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). +#: +#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each +#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. +#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort +#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated +#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of +#: what a view IS: keeping it out of the view config means duplicating or exporting a view does +#: not drag a folder reference along with it. One map, one home, both surfaces. +FOLDER_SURFACES = {"views", "cohorts"} +MAX_FOLDERS = 60 +MAX_FOLDER_NAME = 80 + +#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. +#: +#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already +#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, +#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the +#: client cannot render can never reach the store. +#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — +#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional +#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set +#: without the matching client geometry: an unknown shape falls back to the default folder mark, +#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. +FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} +#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the +#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. +FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} +FOLDER_ICON_DEFAULT_TONE = "grey" + + +#: Wave-9 I17 (contract C4) — who may EDIT a saved view. +#: +#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today +#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one +#: user's views are invisible to every other user and "collaborative" has nothing to act on +#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable +#: and fail-closed now, so that when shared views land the permission does not need a data +#: migration and no saved view is retro-restricted. It does NOT make anything shared, and +#: nothing in the app currently reads it to grant or deny cross-user access. +#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. +VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} +MAX_VIEW_USERS = 50 + + +def clean_view_permissions(raw, default, known_users=None): + """{edit, users?} — fail-closed on both halves. + + `default` is supplied by the CALLER because it splits by path, and that split is a + permission rule rather than a formatting one: absent on a view that already exists means a + pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a + silent takeaway), while absent on CREATE must be 'personal' (a new view must never be + anyone-can-edit purely by omission). + + `known_users` (when given) is the real account list: an unknown name is DROPPED, and an + 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. + """ + mode = (raw or {}).get('edit') if isinstance(raw, dict) else None + if mode not in VIEW_EDIT_MODES: + mode = default if default in VIEW_EDIT_MODES else 'personal' + if mode != 'users': + return {'edit': mode} + names, seen = [], set() + for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: + u = str(u or '').strip() + if not u or u.lower() in seen: + continue + if known_users is not None and u not in known_users: + continue # fail-closed: a name we cannot resolve grants nothing + seen.add(u.lower()) + names.append(u) + if not names: + return {'edit': 'personal'} # an empty grant is NOT "everyone" + return {'edit': 'users', 'users': names} + + +def clean_folder_icon(raw): + """{shape, tone} or None. Fail-closed on both halves, independently. + + A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather + than losing the icon entirely — losing a user's pick because one half was wrong is the kind + of silent data loss the folder events already avoid elsewhere. + """ + if not isinstance(raw, dict): + return None + shape = raw.get("shape") + if shape not in FOLDER_ICON_SHAPES: + return None + tone = raw.get("tone") + return {"shape": shape, + "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} + + +def clean_folders(raw): + """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" + out = {} + for surface in FOLDER_SURFACES: + items, seen = [], set() + for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:80] + name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] + if not fid or not name or fid in seen: + continue # an unidentified or unnamed folder cannot be shown or edited + seen.add(fid) + try: + order = int(f.get("order", len(items))) + except (TypeError, ValueError): + order = len(items) + row = {"id": fid, "name": name, "order": order} + icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark + if icon: + row["icon"] = icon + items.append(row) + items.sort(key=lambda x: x["order"]) + for i, f in enumerate(items): + f["order"] = i # re-index so `order` is always dense and total + if items: + out[surface] = items + return out + + +#: ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, raised by SESSION C as ASK C-16) — "FILED AT ROOT". +#: +#: ⛔ THE DEFECT IS THAT ROOT WAS REPRESENTED BY *ABSENCE*, AND ABSENCE CANNOT HOLD TWO FACTS. +#: "this arrived by grant and was never filed" and "the receiver deliberately dragged this OUT of +#: the Shared group" were the same stored state — nothing — so the client had to GUESS, and +#: `folders.ts::groupByFolder` guessed "Shared". That is why only folder→folder moves appeared to +#: work: **the root bucket was unreachable for a shared view by construction.** +#: +#: ⚠ A RESERVED FOLDER ID, NOT A NEW FIELD, deliberately. The placement map is `{itemId: folderId}` +#: and every reader on both sides already understands it; a parallel "filedAtRoot" set would be a +#: second source of truth for one question, and the two would disagree the first time one of them +#: was written without the other. This id names no folder BY DESIGN and is therefore exempt from +#: the folder-exists test below — it is the one value that means "no folder, on purpose". +#: ⚠ Spelled `ROOT_FOLDER_ID` on the client (`customer-grid/folders.ts`, C's file). Two spellings +#: of one constant is [[a-constant-two-features-share]]; `verify_folders`/`verify_api` assert they +#: agree rather than a comment asking nicely. +ROOT_PLACEMENT = "__root__" + + +def clean_item_folders(raw, folders, valid_ids): + """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. + + This is what makes a deleted folder's contents fall back to the root rather than vanish: + nothing stores "this item is in no folder", so an unresolvable placement simply disappears + and the item renders at the top level. Same for an item that was deleted elsewhere — its + stale placement can never resurrect it, because the sidebars render ITEMS and consult this + map, never the other way round. + + ⭐⭐ WAVE 32 — THE PARAGRAPH ABOVE STATES THE FEATURE AND THE BUG IN ONE SENTENCE, and it took + owner item 20 to notice they were the same mechanism. *"Nothing stores 'this item is in no + folder', so an unresolvable placement simply disappears"* is exactly right for a DELETED FOLDER + (its contents should fall to the root) and exactly wrong for a SHARED VIEW (falling back means + falling back INTO the Shared group, which is where it started). `ROOT_PLACEMENT` is the value + that survives this function so the second case can be said out loud. + """ + out = {} + for surface in FOLDER_SURFACES: + fids = {f["id"] for f in (folders or {}).get(surface, [])} + ok = {} + for item_id, fid in ((raw or {}).get(surface) or {}).items(): + if not isinstance(item_id, str) or not isinstance(fid, str): + continue + # ⛔ `fid == ROOT_PLACEMENT` FIRST, and it is NOT in `fids` — it names no folder, which + # is the whole point. Without this clause the value is written by `item_move` and + # scrubbed here on the way back out, so the mark would be stored and instantly lost: + # the two halves are ONE change and shipping either alone is worse than shipping + # neither ([[lost-write-looks-like-failed-read]]). + if item_id in (valid_ids or {}).get(surface, ()) and (fid == ROOT_PLACEMENT + or fid in fids): + ok[item_id[:120]] = fid + if ok: + out[surface] = ok + return out + + +def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): + """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). + + Returns a clean tree of leaf conditions ({colId, op, value, value2}) and + groups ({conj, children}). Module-agnostic on purpose: any module embedding + the grid validates its own view state through this one function. + + Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — + the same contract the rest of the view sanitiser follows, so one bad rule can + never cost a user their whole saved view. Depth, per-level width and total + node count are all capped: the tree is re-evaluated for every row on every + render, so an unbounded structure would be a persistent client-side DoS. + Empty groups are dropped (they carry no meaning once persisted). + + `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is + DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a + condition that can only match nothing, so `List is not [deleted]` would show an empty table + forever with no way to tell why. `None` means this host has no cohorts, and then every + cohort leaf is dropped: fail-closed, like every other unknown key. + """ + if budget is None: + budget = [MAX_FILTER_NODES] + out = [] + for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: + if budget[0] <= 0: + break + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): # a condition GROUP + if depth >= MAX_FILTER_DEPTH: + continue # too deep -> drop + budget[0] -= 1 + children = clean_filter_tree(node['children'], valid_keys, + depth + 1, budget, cohort_ids) + if children: + out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', + 'children': children}) + continue + if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf + op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) + named = parse_cohort_ids(node.get('value')) + # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT + # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to + # `is none of [A]` would show every row in B under a count nobody would doubt. This + # is the same all-or-nothing the single-cohort leaf already had, extended to a set. + if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): + budget[0] -= 1 + out.append({'colId': COHORT_FIELD, 'op': op, + 'value': ','.join(named), 'value2': ''}) + continue + if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: + budget[0] -= 1 + # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the + # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client + # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently + # stop filtering and show every row instead of the zero-revenue ones. + val, val2 = node.get('value'), node.get('value2') + leaf = {'colId': node['colId'], 'op': node['op'], + 'value': ('' if val is None else str(val))[:500], + 'value2': ('' if val2 is None else str(val2))[:500]} + # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two + # extra members: a stable client-generated `id`, which is how the server's answer + # finds its way back to the condition that asked (positional matching silently + # re-associates every answer the moment a user deletes a condition), and the + # `window`. Emitted ONLY when the input has them — a column condition's cleaned + # shape is unchanged, so every persisted view deserialises byte-identically and + # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by + # exact structural comparison). + rid = node.get('id') + if rid not in (None, ''): + leaf['id'] = str(rid)[:64] + window = _clean_window(node.get('window')) + if window is not None: + leaf['window'] = window + # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted + # only when the input has them, so a plain column condition's cleaned shape is + # byte-identical to what it always was and `clean_filter_tree` stays idempotent + # (verify_filter_engine.py asserts that by exact structural comparison). Drop the + # carry-through and the next autosave silently strips a date condition back to a + # bare comparison against an empty value — i.e. back to INACTIVE. + date_window = _clean_window(node.get('dateWindow')) + if date_window is not None: + leaf['dateWindow'] = date_window + mode = node.get('dateMode') + # SHAPE only. An unrecognised mode survives here and is refused by + # `windows.resolve_anchor`, which returns None and makes the condition match + # NOTHING — the same split as `_clean_window`, and the reason this module can stay + # free of the date vocabulary it would otherwise have to keep in step. + if isinstance(mode, str) and 0 < len(mode) <= 40: + leaf['dateMode'] = mode + rhs = _clean_rhs(node.get('rhs'), valid_keys) + if rhs is not None: + leaf['rhs'] = rhs + out.append(leaf) + return out + + +def _default_view_config(fields): + # ⭐⭐ W30-T41's SERVER HALF (F's ask F-1, answered by D — this file is D's fence). + # + # ⛔ THE SECOND ARM USED TO BE `or field["source"] == "overlay"`, AND IT SWALLOWED THE FIRST + # ONE FOR EVERY CONNECTED COLUMN. `user_tables._clean_field` stamps `source: "overlay"` on + # every `ut_` field, so on an Odoo grid the arm was true for ALL of them and `default: False` + # meant nothing: `odoo_id`, `state`, `customer_link` and `partner_id` opened SHOWN however + # they were declared. The exception had become the rule ([[fallback-that-became-the-rule]]), + # and it is the same predicate `useGridColumns.isDefaultVisible` carried on the client. + # + # ⚠ AND THE TWO HALVES MUST MOVE TOGETHER, which is why this is not cosmetic. `CustomerGrid` + # compares the stored view against its own `defaultViewConfig` by JSON equality; with the + # client fixed (T41) and this left alone, the system view would differ from the client's + # default on every render — a view that looks permanently dirty and autosaves forever, which + # is the failure `verify_filter_engine`'s key-ORDER check exists to prevent, one level down. + # + # ⚠ A USER-CREATED COLUMN IS UNAFFECTED, and that is why the fix is a DELETION rather than a + # carve-out for the four Odoo keys: it carries no `default` key at all, so `is not False` + # keeps it visible. On the main Customer grid exactly one field moves — `notes`, which asks + # to be hidden in its own declaration and was being shown against it. + shown = [field["key"] for field in fields if field.get("default") is not False] + hidden = [field["key"] for field in fields if field["key"] not in shown] + return { + # `filters` is the ROOT of the filter tree: leaf conditions and/or nested + # condition groups ({conj, children}); `filterConj` joins the root level. + "filters": [], "filterConj": "and", + "sorts": [], "groupBy": None, "colorBy": None, + "rowHeightMode": "short", "order": shown + hidden, "visible": shown, + "widths": {}, "memberPids": [], + } + + +#: Wave 17 R1 / C-LOCKV — the `kind` a PROJECTED locked view wears. A cohort is not a separate +#: kind of object any more: it is a saved view whose rows are a hand-curated set. +LOCKED_VIEW_KIND = 'locked' + + +def locked_view_projection(entry, base_config): + """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV). + + ⛔ THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the + view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the + named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do + all the work with no second mechanism. Membership itself is NEVER copied in here — it stays + in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped + collection inside a client-writable `config` is deleted by the next autosave (see the + contract's reason 2). + + ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set: + these views are ordinary in every respect the owner asked for — reorder, folder, sort, + filter, change display mode. The lock mark in the rail is driven by `kind`. + """ + return { + 'id': entry['id'], + 'name': entry.get('name') or entry['id'], + 'kind': LOCKED_VIEW_KIND, + 'config': {**base_config, 'cohortLock': entry['id']}, + } + + +#: ⭐ WAVE-27 item 27 (owner ruling R8) — the IG "Overview" view's CURATED COLUMNS, in the +#: owner's own order: handle, followers, engagement, location, last enriched. +#: +#: Written as candidates rather than as a requirement. The template registry REFUSES a template +#: whose columns the target lacks (`view_templates.missing_columns`) because applying one writes +#: the user's own views and a filter on a missing column silently WIDENS. This view is INJECTED, +#: not applied, and it filters nothing — so the proportionate rule is the opposite one: take the +#: columns the table has, in this order, and skip the rest. An IG database that predates a +#: column simply shows the other four. +#: +#: ⚠ `location_guess` is SESSION B's item-16 column and may not exist yet. That is exactly why +#: this list is intersected rather than asserted: a hard requirement here would make the whole +#: view vanish (or the assembly refuse) on every tenant until B lands, and then appear by +#: surprise. `profile_url` closes the list as the click-through, which is what makes the view +#: usable rather than merely informative. +IG_OVERVIEW_COLUMNS = ('handle', 'full_name', 'followers', 'avg_engagement', + 'location_guess', 'enriched_at', 'profile_url') + +#: The id is PINNED, the `view_templates` discipline: re-assembling must update the same view +#: rather than mint "Overview 2". It also lets a user's own edits overlay it through the saved +#: -config loop below, exactly as a cohort projection does. +IG_OVERVIEW_ID = 'tpl_overview' + +#: How this function recognises an IG preset database WITHOUT importing the engine: two of the +#: profile preset columns is a stronger signal than any single one (a hand-made table could +#: plausibly own a column called `followers`; owning `followers` AND `avg_engagement` AND +#: `handle` is the preset set). `core/` must stay importable without the API layer, so this +#: mirrors `user_tables.PROFILE_PRESET_KEYS` the way that module mirrors the engine's. +_IG_SIGNATURE = ('handle', 'followers', 'avg_engagement') + + +def _ig_overview_view(fields, base): + """R8's curated Overview, or None when this table is not an Instagram one.""" + keys = {f['key'] for f in fields} + if not all(k in keys for k in _IG_SIGNATURE): + return None + visible = [k for k in IG_OVERVIEW_COLUMNS if k in keys] + return { + 'id': IG_OVERVIEW_ID, + 'name': 'Overview', + 'kind': 'system', + # NOT `locked`. The system view is locked because it is the identity of the table ("show + # me everything"); this one is a STARTING LAYOUT, and R8 calls it curated rather than + # fixed. A user who wants a sixth column should get one. + 'note': 'The five things worth seeing first on a creator. Sorted by reach.', + 'config': { + **dict(base), + 'visible': visible, + 'order': visible + [k for k in (f['key'] for f in fields) if k not in visible], + 'sorts': ([{'colId': 'followers', 'dir': 'desc'}] + if 'followers' in keys else []), + }, + } + + +def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None, + view_order=None): + """Convert legacy list formulas into the shared serializable SavedView contract. + + `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the + product surface). The ID stays "all-customers" on every topic — the client pins it + (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork + that contract for a label's sake. + + `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose + id IS the cohort id — so every stored reference to that id (a `cohortLock` on another view, + an `is part of` condition, a folder placement) keeps pointing at the same thing and no + rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the + `list:` views have always used, which is what gives a locked view its own sort, filter, + columns and display mode with no new storage.""" + base = _default_view_config(fields) + views = [{ + "id": "all-customers", "name": system_name, "kind": "system", + "locked": True, "config": dict(base), + }] + # ⭐ WAVE-27 item 27 (R8) — the IG Overview, ABOVE All records. + # + # ⚠ INJECTED, not seeded into the store, and that is what makes "existing IG databases gain + # it too" true with no migration and no write on a read path. It is the same mechanism the + # system view above has always used; the pinned id means a user's own edits overlay it + # through the saved-config loop below rather than forking a second view. + _overview = _ig_overview_view(fields, base) + if _overview: + views.insert(0, _overview) + op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", + "contains": "contains"} + for name, definition in (defs or {}).items(): + filters = [] + for rule in definition.get("rules") or []: + if rule.get("field") not in {field["key"] for field in fields}: + continue + filters.append({ + "colId": rule["field"], + "op": op_map.get(rule.get("op"), "eq"), + "value": str(rule.get("value") if rule.get("value") is not None else ""), + }) + sort = str(definition.get("sort") or "") + sorts = ([{"colId": sort.lstrip("-"), + "dir": "desc" if sort.startswith("-") else "asc"}] + if sort.lstrip("-") in {field["key"] for field in fields} else []) + views.append({ + "id": "list:" + str(name), + "name": str(name), + "kind": "list", + "note": str(definition.get("note") or ""), + "config": { + **base, "filters": filters, "sorts": sorts, + "memberPids": [int(pid) for pid in definition.get("members") or [] + if isinstance(pid, int) or str(pid).isdigit()], + }, + }) + # Wave 17 R1 — the cohorts, as ordinary views. Appended BEFORE the saved-config overlay + # below so a user's own edits to a locked view (its sort, its columns, its display mode) + # land on the projection instead of creating a second row with the same id. + for _entry in (locked_lists or []): + if isinstance(_entry, dict) and _entry.get('id'): + views.append(locked_view_projection(_entry, base)) + index = {view["id"]: i for i, view in enumerate(views)} + for view_id, saved in (saved_views or {}).items(): + if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): + continue + clean = dict(saved) + clean["id"] = str(view_id) + if view_id in index: + views[index[view_id]] = clean + else: + views.append(clean) + # ⛔ WAVE 17 R1 — RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected + # view with its saved record, and a saved record that omits `cohortLock` would hand back a + # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the + # client rebuilds `config` on every autosave (a column resize is enough), and the lock is + # identity here, not something the browser is the source of truth for. Read-side rather than + # write-side-only on purpose — this also repairs any record already written by another path. + _locked_ids = {e['id']: e for e in (locked_lists or []) + if isinstance(e, dict) and e.get('id')} + if _locked_ids: + for _v in views: + _entry = _locked_ids.get(_v.get('id')) + if not _entry: + continue + _v['kind'] = LOCKED_VIEW_KIND + _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']} + # One thing, one name: the cohort store owns it (the rename event routes there), so + # a stale `name` on the saved record can never fork into a second title. + _v['name'] = _entry.get('name') or _v['id'] + # ── ⭐ WAVE-27 item 5, contract C7: the PER-USER VIEW ORDER ─────────────────────────────── + # + # `view_order` is a list of view ids this user dragged into place. Applied LAST, over the + # finished list, so it reorders whatever the assembly produced without having to know how any + # of it got there (system, list:, cohort projection, saved, injected Overview). + # + # ⛔ THE SYSTEM VIEW STAYS AT INDEX 0 (C7), and it is re-pinned here rather than trusted to + # sort correctly: `all-customers` is the client's landing default and one of its + # UNDELETABLE_VIEW_IDS, so a stored order that happened to omit it — or list it third — + # would move the rail's home row. ⚠ R8's Overview is the ONE thing allowed above it, because + # the owner put it there; it is re-pinned with the system view so a drag cannot bury it + # either. Both are facts about the table rather than the user's arrangement of it. + # + # ⚠ UNKNOWN IDS APPEND IN SERVER ORDER (C7). A view created since this order was stored, or + # one shared to this user yesterday, must APPEAR — dropping it would make sharing look + # broken, and the failure would be invisible to whoever shared it. Ids in the stored order + # that no longer resolve are simply skipped. + if view_order: + _rank = {vid: i for i, vid in enumerate(view_order) if isinstance(vid, str)} + _pinned = [v for v in views if v.get('id') in (IG_OVERVIEW_ID, 'all-customers')] + _rest = [v for v in views if v.get('id') not in (IG_OVERVIEW_ID, 'all-customers')] + # A stable sort over a rank that DEFAULTS TO THE END keeps unranked views in their + # server order behind the ranked ones, rather than interleaving them by accident. + _rest.sort(key=lambda v: _rank.get(v.get('id'), len(_rank) + 1)) + views = _pinned + _rest + return views + + +def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, + fields_base=None, with_cohorts=True): + """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE + projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`). + + ⛔ WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no + `storageKey` — and the client validator (`fetchWorkspace`) requires one, so the standalone + shell silently discarded the whole workspace: saved views never rendered and `cohortMode` + never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline + projection into the route would have re-created the same drift one wave later; extracting it + means the wire can only be one thing. + + Returns `(workspace, fields, views, cohort_lists)` — the extra three because the host + interleaves further work (docs, derived cells, measure sets) that consumes them. + + HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`, + `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`). + + Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer, + byte-identical). + + ⭐ WAVE 19 / R9 — `with_cohorts` NO LONGER MEANS "customer only". Wave 16 set it False on the + product surface because cohorts were a single customer-keyed bucket, so resolving them against + product pids would have intersected two unrelated id spaces and printed a plausible, + meaningless member count. `modules.cohort` is scope-parameterized now: the lists come from + THIS topic's bucket (`cohort_mod.scoped(scope_key)`), so their ids are this topic's ids and + the intersection with `pool_pids` is the ordinary one. The flag survives as an honest OFF + switch for a surface that wants no membership channel at all — it is not a topic wall. + """ + import modules.cohort as cohort_mod + + cohort_lists = [] + if with_cohorts: + for cid, c in sorted(cohort_mod.scoped(scope_key).visible(uname, pool_pids).items(), + key=lambda kv: (kv[1].get('name') or '').lower()): + members = [p for p in (c.get('members') or []) if p in pool_pids] + entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members} + # Rule 8b: a member can drop out of the 24-month pool without the cohort being + # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule + # forbids. + missing = len(c.get('members') or []) - len(members) + if missing: + entry['missing'] = missing + cohort_lists.append(entry) + + fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key, + fields_base=fields_base) + views = views_from_defs(defs or {}, ws.get('views'), fields, + # Wave 21 (item 3, R6): the system view's name is TOPIC-DERIVED. A + # user database's default view used to read "All customers" — a + # compiled customer literal minted on every topic, one half of the + # owner's "my new database looks like RI's Customer table". The ID + # stays 'all-customers' everywhere (pinned client+server — the + # client's UNDELETABLE set and the view pin both name it). + system_name=("All products" if scope_key == 'product' + else "All records" + if str(scope_key or '').startswith('ut_') + else "All customers"), + locked_lists=cohort_lists, + # ⭐ WAVE-27 item 5 (C7) — this user's own rail arrangement, from + # their own stratum. Read here rather than sorted by the client so + # the ORDER a request answers with is the order that was stored: + # sorting client-side would make the rail settle after a paint on + # every load, and shared views would land in server order first. + view_order=ws.get('viewOrder')) + workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists} + # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy + # wins when present; this is the server's answer for a FRESH browser, which used to fall + # all the way to the system default view (and whatever display mode was stored on it). + if ws.get('activeViewId'): + workspace['activeViewId'] = str(ws['activeViewId']) + + # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be + # deleted by a path that knows nothing about folders, and the placement map must not outlive + # the thing it points at. + _folders = clean_folders(ws.get('folders')) + _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')} + # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT + # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so + # its folder has to be an ordinary view folder — but rewriting the stored map would be a + # one-shot write that has to be got right once, while this is a projection that is right + # every time it runs. New drags write to `views` anyway (the client only knows that + # surface), so `cohorts` drains on its own and never needs a second pass. + # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged: + # re-parenting somebody's list into a folder that merely shares an id is a worse outcome + # than the list appearing at the root, where it is visible and one drag from home. + _cf = list(_folders.get('cohorts') or []) + if _cf: + _vf = list(_folders.get('views') or []) + _taken = {f['id'] for f in _vf} + _order = len(_vf) + for _f in _cf: + if _f['id'] in _taken: + continue + _vf.append({**_f, 'order': _order}) + _order += 1 + _folders['views'] = _vf + _raw_item_folders = dict(ws.get('itemFolders') or {}) + if _raw_item_folders.get('cohorts'): + # Cohort placements now describe VIEWS (same ids — that is the point of preserving them). + # A placement already stored on the views surface WINS: it is the more recent act. + _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}), + **dict(_raw_item_folders.get('views') or {})} + _placed = clean_item_folders( + _raw_item_folders, _folders, + {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}}) + if _folders.get('views'): + workspace['folders'] = _folders['views'] + # ⛔ `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a + # wire that still described one would invite a second rendering of rows that are now views. + _vplaced = _placed.get('views') or {} + for _v in (views or []): + if isinstance(_v, dict) and _v.get('id') in _vplaced: + _v['folderId'] = _vplaced[_v['id']] + _cplaced = _placed.get('cohorts') or {} + for _c in cohort_lists: + if _c['id'] in _cplaced: + _c['folderId'] = _cplaced[_c['id']] + + # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order, + # re-validated at SERVE time exactly like folders — a field can be deleted by a path + # that knows nothing about this stratum, and a stale key must not outlive its field. + _rl = ws.get('recordLayout') + if isinstance(_rl, dict) and isinstance(_rl.get('order'), list): + _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')} + _order, _seen = [], set() + for _k in _rl['order'][:200]: + _k = str(_k or '') + if _k and _k in _fkeys and _k not in _seen: + _seen.add(_k) + _order.append(_k) + if _order: + workspace['recordLayout'] = {'order': _order} + + return workspace, fields, views, cohort_lists + + +def embed_html_path(): + """The first existing candidate path for the inlined single-file build, or None.""" + for p in _EMBED_CANDIDATES: + if p.is_file(): + return p + return None + + +def scope_counts(shown, matched, total): + """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). + + `matched` and `total` MUST come from their own queries over the whole scope. Never pass + `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would + report the window size as though it were the result size. + + Refuses the shapes that could only be a mistake, because a wrong count here is invisible on + screen (it looks like a smaller dataset, not like an error). + """ + shown, matched, total = int(shown), int(matched), int(total) + if matched > total: + raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " + f"more rows than the scope holds") + if shown > matched: + raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " + f"more rows than the filter matched") + return {"shown": shown, "matched": matched, "total": total, "windowed": True} + + + +# ⛔ EXIT-6 (2026-08-04): `build_html`, `component_dir`, `render` and `_DECLARED_COMPONENTS` WERE +# HERE, and they are gone with Streamlit. They were the EMBED HOST — the path that declared the +# prebuilt bundle as a `streamlit.components.v1` custom component (or injected the single-file +# HTML build as a fallback) so the React grid could be drawn inside a Streamlit page. +# +# THIS MODULE ITSELF SURVIVES, and that distinction is the whole point: `aios_grid.py` is imported +# at 11 sites across `aios-web/api/` plus `harness/semantic.py` — it owns the canonical field +# contract, the workspace wire and the count envelope. Only the ~95 lines that knew about a HOST +# went; the rest never did. Its one and only `import streamlit` lived inside `render`, lazily, and +# left with it. `api/verify_no_streamlit.py` now gates that nothing here re-imports it. +# +# Deleted with them: `aios_grid_embed.html` + `aios_grid_component/index.html` (a 2.1 MB prebuilt +# bundle), `build_embed.py` that produced them, and `deploy_hf.py`'s embed-staleness guard. The +# React app is now served directly by the FastAPI container — there is no twin to keep fresh, so +# the entire class of "the code shipped but the bundle did not" is retired rather than guarded. diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index 3944187b70b630f840161399751e6f7fabb9a979..cef82e7bf760270e0a753bb4a535868ff446e1b0 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -122,6 +122,19 @@ def hidden_keys(user, module, fields): ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate anyway; without it a self-referential pair would spin here. + + ⭐⭐ W36-T21 — AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this + closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup + names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that + link points at — so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices + this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns + what the hidden link contains, in aggregate; the three outcomes are exactly the three the + formula argument above enumerates, and only "strip both" is coherent. Verified against the + real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there + is either `{'link': , 'field': }` + or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes + and `rollup.field` is deliberately not treated as one — it names another database's column, + which has its own wall. """ e = entry(user, module) if perms.is_admin(user) or not e: @@ -137,6 +150,9 @@ def hidden_keys(user, module, fields): expr = f.get('formula') if isinstance(expr, str) and expr: refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()} + link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None + if isinstance(link, str) and link.strip(): + refs.setdefault(f['key'], set()).add(link.strip()) MAX_PASSES = 12 for _ in range(MAX_PASSES): @@ -452,3 +468,336 @@ def _group_dba_team(group): if vals <= allowed: return tid return None + + +# ── C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ──────────────────────────────── +#: ⭐⭐ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same +#: permission logic, always"* — per-user field visibility AND row filtration on every database +#: carrying a unique id, whatever created it, with a NEW database inheriting it by construction +#: rather than by a list somebody maintains. +#: +#: ⛔ THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line +#: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic +#: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone — +#: creator, admin, or a `core.shares` grant — which is a BINARY door: you see all 31,418 rows of +#: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this +#: work in as many words (*"Arming `perm_scope` over `ut_*` … booked, not faked"*), and owner +#: item 11 is that booking coming due. +#: +#: ⚠ AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT +#: user-created. Ten of them (`ut_odoo_invoices`, `…_orders`, `…_agents`, `…_accounts`, `…_bills`, +#: `…_vendors`, `…_order_lines`, `…_gl_lines`, `…_customers`, `…_products`) are generated by the +#: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a +#: statement about origin**, and a wall keyed off it was reading a naming artefact as a security +#: boundary. +#: +#: ⛔⛔ THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and +#: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES +#: them — it calls `may_open`, it does not reimplement it — because merging them is how this +#: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20 +#: note). One resolver per question, asked in order. + + +class UnknownTable(LookupError): + """No database in this tenant answers to that key. + + ⛔ RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this + database is empty"* — indistinguishable from a real empty table, and the caller least able to + notice is the one that wanted rows. This repo has shipped that exact silent-empty answer + before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]). + """ + + +class Denied(PermissionError): + """This principal may not read this database at all. The IF question, answered by `may_read`.""" + + +class Unresolvable(RuntimeError): + """The rows exist and cannot be served under this call's constraints — R6's SECOND SENTENCE. + + ⭐ STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there + is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a + limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never + silently enforced as a short answer. Carries the same four keys + `routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight + to a client without a second vocabulary ([[one-question-two-normalizers]]). + """ + + def __init__(self, subject, effect, cause, recommendation): + self.subject, self.effect = subject, effect + self.cause, self.recommendation = cause, recommendation + super().__init__(f"{subject}: {effect}. {cause}. {recommendation}") + + def as_limit(self): + """The dict shape `routes_tables` puts in an assembly's `limits` list.""" + return {"subject": self.subject, "effect": self.effect, + "cause": self.cause, "recommendation": self.recommendation} + + +#: Row readers DECLARED by the app layer, keyed by EXACT database key. +#: `reader(table_key, user, st) -> (fields, rows)`. +#: +#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and +#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache +#: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables` +#: already uses for exactly this reason — `register_connected`, `register_read_through`, +#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being +#: interrogated by it."* +_ROW_SOURCES = {} + +#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. +#: `reader(table_key, field_keys, st) -> rows`. +_MIRROR_READER = None + + +def register_rows(reader, *table_keys): + """Declare who reads a NAMED database's rows. Returns the registered key set. + + ⚠ The return value is the registrar's own answer on purpose: a public function whose only + caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that + says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write + door's return keeps one construction site of the set instead of two. + """ + for key in table_keys: + k = str(key or '').strip() + if k: + _ROW_SOURCES[k] = reader + return frozenset(_ROW_SOURCES) + + +def register_mirror(reader): + """Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`).""" + global _MIRROR_READER + _MIRROR_READER = reader + return _MIRROR_READER is not None + + +#: ⛔ `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE +#: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE +#: list to read"* — and `register_rows` ALREADY returns exactly that, which is the same idiom +#: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through +#: the write door's return keeps ONE construction site of the set. A second accessor beside it is a +#: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py` +#: — the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by +#: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the +#: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered. + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def may_read(user, table_key, st=None): + """May this principal read `table_key` AT ALL — the IF question, on EVERY database. + + ⛔ COMPOSED, NOT RE-DERIVED, and the order is the whole rule: + + 1. an admin reads everything (break-glass — `deps._user_for` hands back a hardcoded master + dict on a store outage and it will never carry a `perms` block); + 2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner + item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall + below would admit, and it may never grant one that wall refuses; + 3. a `ut_*` database defers to `user_tables.may_open` — creator, admin, or a `core.shares` + grant — UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."* + 4. anything else is a registry topic and defers to `may_access` above. + + ⛔⛔ WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does. + `may_access` reads migrated-and-undeclared as DENY — correct for a topic, because + `routes_admin` writes an entry for every governable topic on every save. ⚠ NO `ut_*` ENTRY WAS + STORABLE AT ALL UNTIL W36-T22 — `_clean_perms` refused the key with `unenforced_module` — so + every record migrated before this wave carries no entry for any of them, and reading that + absence as a decision would revoke all ten keychain databases from every migrated account the + moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been + answering rather than the marker that has not, and it keeps being right AFTER the flag's + deletion: an admin who has never opened the editor for a database has still not decided + anything about it. + + ⛔⛔ PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a + stored record is keyed BY username in that bucket and does not carry one INSIDE it — only + `core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand + this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*` + database reads as DENIED for an account that can open all of them. It fails in the SAFE + direction and is silently wrong, which is the worst pair to debug — it cost two call sites in + one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage. + """ + if perms.is_admin(user): + return True + e = entry(user, table_key) + if e is not None and not bool(e.get('access', True)): + return False + key = str(table_key or '') + if key.startswith(_ut().KEY_PREFIX): + return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) + return may_access(user, table_key) + + +def wall_declared(user, table_key): + """Is a ROW or FIELD narrowing declared for this principal on this database? + + ⛔ THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a + stored `ut_*` wall would be INERT — *"the editor would say DENY, the table routes would keep + serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore + REFUSE for a principal this returns True for, rather than serve the whole database. False for + an admin (they bypass the wall entirely) and for any record with no entry, so a door asking + this pays nothing and changes nothing for everybody who has no wall. + """ + if perms.is_admin(user): + return False + e = entry(user, table_key) + if not e: + return False + return bool(e.get('filter')) or bool(e.get('hiddenFields')) + + +def row_scope_applies(user, table_key): + """Does a permanent ROW filter narrow this principal on this database? + + ⚠ `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows + it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its + whole point is that the pid set costs no row pass, and paying for one on every database + switch — for every account, walled or not — would undo W30-T30 to enforce a rule that applies + to almost nobody. Asked here rather than spelled out at the call site, so there is ONE + statement of when the row wall bites ([[one-question-two-normalizers]]). + """ + if perms.is_admin(user): + return False + return bool((entry(user, table_key) or {}).get('filter')) + + +def scoped_table(user, table_key, st=None, ctx=None): + """⭐⭐ CONTRACT C1 — the rows of ANY database, already field-stripped and row-filtered for + `user`. Registry topic or `ut_*`; there is no third kind and no per-database branch. + + rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database + rows = scoped_table(user, 'customer_data') # a registry topic + + `user` is a user RECORD (the dict `deps.Session.user` carries), not a username — the whole + wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller + that forgets the principal must not run, because the only thing a defaulted one could mean is + "unscoped", which is the widening direction. + + ⛔ FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the + right status instead of guessing: `UnknownTable` (no such database — never an empty list), + `Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is + why — standing rule 1's second sentence). + + ⚠ NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only + thing that stops it is a population that exceeds one materialisation window, and that arrives + as `Unresolvable` carrying its cause and a recommendation rather than as a short answer. + + ⭐ E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is + THE door rather than A door: everything a sandboxed script may read, it reads here, under the + CALLING user's scope (R5). + """ + _fields, rows = _scoped(user, table_key, st=st, ctx=ctx) + return rows + + +def scoped_fields(user, table_key, st=None): + """The COLUMNS of any database this principal may see — C1's other half. + + ⛔ IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the + field list and the row payload are two different wires, and narrowing one without the other + leaves the value sitting where anything can read it. A caller that must render a scoped table + needs both, and E cannot read a `ut_*` definition to learn its columns — the sandbox has no + second path to the store (W1). So both come from here, off one wall. + + ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a + registry topic it goes through the registered reader, which builds that topic's pool; the + pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. + """ + fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) + return fields + + +def _scoped(user, table_key, st=None, ctx=None, want_rows=True): + """`(fields, rows)` — ONE evaluator behind both public doors, so they cannot disagree.""" + key = str(table_key or '').strip() + if not key: + raise UnknownTable('a database key is required. This door will not guess which database ' + 'was meant') + if not may_read(user, key, st=st): + raise Denied(f"this account may not read '{key}'") + fields, rows = _read(key, user, st, want_rows) + # THE FIELD WALL — a TRANSITIVE closure, so hiding a column also hides every formula computed + # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference + # is the wrong shape here. + hide = hidden_keys(user, key, fields) + if not want_rows: + return (visible_fields(fields, user, key) if hide else fields), [] + # THE ROW WALL — `permits()`, so a permanent filter this evaluator cannot answer DENIES + # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a + # permanent filter may name a column the reader is not allowed to SEE, and dropping the + # predicate would widen the read rather than narrow it. + rows = apply_row_scope(rows, user, key, fields, ctx) + if hide: + fields = visible_fields(fields, user, key) + rows = [strip_row(r, hide) for r in rows] + return fields, rows + + +def _read(table_key, user, st, want_rows=True): + """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" + reader = _ROW_SOURCES.get(table_key) + if reader is not None: + fields, rows = reader(table_key, user, st) + return list(fields or ()), list(rows or ()) + ut = _ut() + if not table_key.startswith(ut.KEY_PREFIX): + # ⛔ A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never + # imported the API layer this is the honest answer: nothing here can build that pool. + raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader " + f"is registered for it") + return _read_user_table(table_key, st, want_rows) + + +def _read_user_table(table_key, st, want_rows=True): + """core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately. + + ⭐ WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same + argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold + process — E's sandbox subprocess, a worker, a gate — that never imported an API route still + owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and + the sandbox is exactly such a process. + + ⚠ THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions + without the 28.6 MB of rows (D-213), which is every read this function makes when + `want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so + the materialised arm below takes the whole read explicitly. + """ + ut = _ut() + lent = ut.lend_defs(st) + defn = ut.get(table_key, st=lent) + if not defn: + raise UnknownTable(f"no database named '{table_key}'") + fields = [dict(f) for f in (defn.get('fields') or [])] + if not want_rows: + return fields, [] + if not ut.materialises(table_key, st=st, defn=defn): + # A read-through grid stores no rows here — they live in the mirror, and reading + # `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking, + # wrong, and silent. + if _MIRROR_READER is None: + raise Unresolvable( + subject='rows', effect='unreadable', + cause=(f"'{table_key}' is served read-through from the connector mirror and no " + f'mirror reader is registered in this process'), + recommendation=('call `perm_scope.register_mirror(...)` from the app layer before ' + 'reading a read-through database, or read it through the API')) + keys = {f['key'] for f in fields if f.get('key')} + return fields, list(_MIRROR_READER(table_key, keys, st) or ()) + whole = ut.get(table_key, st=st) + if whole is None: + # Deleted between the wall and here. The same refusal, not an empty table. + raise UnknownTable(f"no database named '{table_key}'") + field_keys = {f['key'] for f in fields if f.get('key')} + rows = [] + for rid, row in (whole.get('rows') or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r['pid'] = int(rid) + rows.append(r) + rows.sort(key=lambda r: r['pid']) + return fields, rows diff --git a/platform/core/perms.py b/platform/core/perms.py index 0c26d8098d989330c5259c53943a17fead2087dc..2a26eeb897862cd97d4b5d6484c3056dff97e566 100644 --- a/platform/core/perms.py +++ b/platform/core/perms.py @@ -269,19 +269,29 @@ def tenant_governable_modules(runtime, topics, ut_entries=()): in; this function is the only place the tenant is applied. `ut_entries` — this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape. - Returns `[{'key', 'label', 'enforced'}]`. - - ⛔⛔ `enforced` IS THE HONEST HALF AND IT IS NOT DECORATION. `perm_scope` walls the REGISTRY - topics — `grid_assembly` calls `may_access`, `visible_fields` and `apply_row_scope` on every - read. It is never consulted for a `ut_*` database: `routes_tables`' visibility is - `user_tables.may_open` (creator, admin, or a `core.shares` grant) and there is no module - grant in that path at all. So a stored `ut_*` wall would be INERT — the editor would say - DENY, the table routes would keep serving, and nothing anywhere would say so. That is the - same class as the empty-dropdown defect (wave 26 item 24): a control that answers when it - cannot. The database is LISTED, because the owner asked to see every database; the wall is - declared unenforced, and `routes_admin::_clean_perms` REFUSES to store one against it rather - than accepting a rule nobody applies. Arming `perm_scope` over `ut_*` is a change to - `routes_tables`/`user_tables` (lane A's fence) — booked, not faked. + Returns `[{'key', 'label'}]`. + + ⭐⭐ WAVE 36 (W36-T22 / CONTRACT C2 / OWNER RULING R6) — **`enforced` IS DELETED, NOT DEFAULTED + TO `True`.** This docstring carried the paragraph that booked the work, and it read: + + *"`perm_scope` walls the REGISTRY topics … It is never consulted for a `ut_*` database … + So a stored `ut_*` wall would be INERT — the editor would say DENY, the table routes would + keep serving, and nothing anywhere would say so … Arming `perm_scope` over `ut_*` is a + change to `routes_tables`/`user_tables` — booked, not faked."* + + That change is W36-T21 and it has landed: `perm_scope.scoped_table` is the ONE door to any + database's rows, `routes_tables` applies the row filter and the hidden-field closure on every + `ut_*` read, and a door that cannot apply them REFUSES rather than serving the lot. So there + is no longer a second class of database for the flag to distinguish, and **a flag that is + always true is a lie with a green gate behind it** — which is why C2 says DELETE rather than + default. Owner item 11, verbatim: *"EVERY database should be able to be toggleable by admin … + Everything that is a database with Unique ID, is always going to be configurable with regards + to Fields, and Filtration per User. Make this infrastructure robust. And always true when we + want to onboard more databases into our App."* + + ⚠ AND THE LIST STAYS DERIVED. Topics come from the registry, `ut_*` from the tenant's own + `nav_entries` — so a database created after this wave is governable with no code change and no + list edit. That is the half of R6 a flag could never have delivered. """ out = [] for t in topics or (): @@ -293,20 +303,23 @@ def tenant_governable_modules(runtime, topics, ut_entries=()): # "Odoo products") would have moved one of them and left the permission editor showing the # old name with nothing to notice. label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key) - out.append({'key': key, 'label': label, 'enforced': True}) + out.append({'key': key, 'label': label}) seen = {r['key'] for r in out} for e in ut_entries or (): key = str((e or {}).get('key') or '').strip() if not key or key in seen: continue seen.add(key) - out.append({'key': key, 'label': (e.get('label') or key), 'enforced': False}) + out.append({'key': key, 'label': (e.get('label') or key)}) return out -def enforced_module_keys(modules): - """The subset of `governable_modules`' answer a permission block may actually be stored for.""" - return {m['key'] for m in (modules or ()) if m.get('enforced')} +#: ⛔ `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key". +#: It read `{m['key'] for m in modules if m.get('enforced')}`, so with the flag gone it would +#: answer the EMPTY SET and `_clean_perms` would refuse every module — and "fix" it by returning +#: everything is a function whose answer no longer depends on its argument, i.e. the always-true +#: flag wearing a different costume. Its one caller (`routes_admin._enforced_keys`) goes with it; +#: what replaces both is the module list itself, which is the only list there is now. def landing_page(user): diff --git a/platform/core/script_sandbox.py b/platform/core/script_sandbox.py new file mode 100644 index 0000000000000000000000000000000000000000..babf60594d17d540e348efa836e40056260a1f96 --- /dev/null +++ b/platform/core/script_sandbox.py @@ -0,0 +1,519 @@ +"""core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python. + +Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they +want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this +script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was +stated, so this file is the guardrail, and one engine serves both items. + +════════════════════════════════════════════════════════════════════════════════════════════════ +⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE. + +In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated +namespace stops import, file, network and environment access — but it **cannot cap memory and +cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow +request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**: +`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the +allow-list inside. Neither half is sufficient; both are load-bearing. + +⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under +the CALLING user's record and serialises the result; the child imports nothing from this repo and +holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store +path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child +reports its own `sys.modules`, and no `core.*` name may appear in it. + +⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute +names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and +the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute +lookup inside `format`, so there is no `ast.Attribute` node to refuse). +════════════════════════════════════════════════════════════════════════════════════════════════ + +The two layers, and they refuse DIFFERENT things on purpose: + + 1. `check_source()` — a pure function over source text. Refuses a construct the language offers + and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every + attribute name outside `ALLOWED_ATTRS`. + 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`, + `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets + past layer 1 still finds nothing to call. + +⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is +refused twice, so a negative control that drops ONE layer sees the other refuse and reports +green — the shape that already cost this wave one missed control in `routes_agent_harness`. So +each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings, +and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching +boundary goes red. +""" +import ast +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform. +DEFAULT_TIMEOUT_S = 10.0 + +#: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report. +DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024 + +#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the +#: wall-clock kill is the primary control and this is the backstop for a child that stops being +#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal. +DEFAULT_CPU_SECONDS = 15 + +#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer, +#: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything +#: that escaped far enough to write to fd 1 has nowhere for it to land. +MAX_STDOUT_BYTES = 64 * 1024 + +#: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule +#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()` +#: returns a named limit carrying its cause and a recommendation. +MAX_PAYLOAD_BYTES = 32 * 1024 * 1024 + +#: The emitted spec. A render spec is a description of a picture; one larger than this is data +#: pretending to be a description. +MAX_SPEC_BYTES = 2 * 1024 * 1024 + +MAX_SOURCE_BYTES = 128 * 1024 + + +# ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════ +#: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` / +#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its +#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__` +#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every +#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never +#: awaited is a silent no-op that looks like a working script). +ALLOWED_NODES = frozenset(""" +Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise +If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword +BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name +Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension +Load Store Del +And Or Not Invert UAdd USub +Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult +Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn +""".split()) + +#: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is +#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are +#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace, +#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__` +#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this +#: sandbox has not been asked for. +#: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup +#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are +#: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused. +ALLOWED_ATTRS = frozenset(""" +append extend insert pop remove clear sort reverse copy count index +keys values items get setdefault update +add discard union intersection difference issubset issuperset +join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold +replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix +isdigit isalpha isalnum isspace isupper islower isnumeric +real imag numerator denominator +""".split()) + + +class Refused(Exception): + """A named refusal: `code` for a caller to branch on, `message` for a person to read.""" + + def __init__(self, code, message): + self.code, self.message = code, message + super().__init__(f"{code}: {message}") + + +def _attr_ok(name): + """An attribute name passes only if it is on the list AND is not private. + + ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every + private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by + accident, which is the one edit a future reader is most likely to make in a hurry. + """ + return name in ALLOWED_ATTRS and not name.startswith("_") + + +def check_source(source): + """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`. + + ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no + process and touches no store, so a gate can hand it a hundred hostile strings for free and an + NC can drop one entry from one frozenset and watch exactly this function change its answer. + """ + text = str(source or "") + if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + return Refused("source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source") + try: + tree = ast.parse(text) + except SyntaxError as exc: + return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}") + + for node in ast.walk(tree): + kind = type(node).__name__ + if kind not in ALLOWED_NODES: + return Refused("refused_construct", + f"line {getattr(node, 'lineno', 0)}: this sandbox does not run " + f"{_english(kind)}") + if isinstance(node, ast.Attribute) and not _attr_ok(node.attr): + return Refused("refused_attribute", + f"line {getattr(node, 'lineno', 0)}: the attribute " + f"'{node.attr}' is not available inside a script view") + # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a + # script that could bind one could shadow the machinery it runs on top of. + if isinstance(node, ast.Name) and node.id.startswith("_"): + return Refused("reserved_name", + f"line {getattr(node, 'lineno', 0)}: names starting with an " + f"underscore are reserved by the sandbox") + if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str( + getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"): + return Refused("reserved_name", + f"line {getattr(node, 'lineno', 0)}: names starting with an " + f"underscore are reserved by the sandbox") + if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"): + return Refused("reserved_name", + f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with " + f"an underscore are reserved by the sandbox") + return None + + +_ENGLISH = { + "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition", + "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop", + "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield", + "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement", + "Match": "a match statement", +} + + +def _english(kind): + return _ENGLISH.get(kind, f"a {kind} expression") + + +# ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════ +#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child. +#: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input` +#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help` +#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step +#: on a published escape, and none has ever been asked for by a script that shapes rows. +#: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause +#: cannot name what it catches is a construct that reads as supported and is not. They are safe +#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this +#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway. +SANDBOX_BUILTIN_NAMES = ( + "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex " + "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed " + "round set slice sorted str sum tuple zip True False None " + "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError " + "AttributeError StopIteration OverflowError" +).split() + +#: The literal program the child runs. It is TEXT rather than a module because the child must +#: import nothing from this repo: a module would be found on `sys.path` and would drag `core` +#: with it, which is exactly the second store path W1 forbids. +#: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so +#: the runner's own machinery cannot be shadowed by the source it executes. +_RUNNER = r''' +import json as _json, os as _os, sys as _sys + +_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read()) +_out = {"ok": False, "code": "not_run", "message": "the script did not run", + "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}} + +# ── the caps this platform can actually apply, reported either way (standing rule 1) ────────── +try: + import resource as _res + _mem = int(_pay["memoryBytes"]) + _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem)) + _out["caps"]["memory"] = True + _cpu = int(_pay["cpuSeconds"]) + _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu)) + _out["caps"]["cpu"] = True +except Exception: + # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says + # which of the three held, never a silent partial. + pass + +_printed = [] +_spent = [0] +_LIMIT = int(_pay["maxStdout"]) + + +def _print(*_a, **_k): + _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n") + _room = _LIMIT - _spent[0] + if _room > 0: + _printed.append(_text[:_room]) + _spent[0] += len(_text) + + +class _Refusal(Exception): + """The SANDBOX refusing, as distinct from the SCRIPT failing. + + Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the + script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's + own code. Two different facts, two different codes. + """ + + +_emitted = [] + + +def _emit(_spec): + if not isinstance(_spec, dict): + raise _Refusal("emit() takes a view spec, which is a dictionary") + if _emitted: + raise _Refusal("emit() was already called; a script view emits exactly one view") + _emitted.append(_spec) + + +_rows = _pay["rows"] +_fields = _pay["fields"] +_bound = _pay["table"] + + +def _scoped_table(_table=None): + if _table is not None and str(_table) != _bound: + raise _Refusal( + "this script view is bound to the database '" + _bound + "' and asked for '" + + str(_table) + "'. A script view reads its own database only") + return [dict(_r) for _r in _rows] + + +def _scoped_fields(): + return [dict(_f) for _f in _fields] + + +_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict) + else getattr(__builtins__, _n) + for _n in _pay["builtins"]}} +_ns["__builtins__"]["print"] = _print +_ns["print"] = _print +_ns["emit"] = _emit +_ns["scoped_table"] = _scoped_table +_ns["scoped_fields"] = _scoped_fields +_ns["table"] = _bound + +try: + exec(compile(_pay["source"], "