diff --git a/RELEASES.json b/RELEASES.json index 477c44cb8fa4e87e2f6addd0de366ee0801eab51..7b55911021fe1eb753f31aef2271e6e60f5f04ed 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,5 +1,5 @@ { - "current": "9445b77", + "current": "2de35b7", "releases": [ { "version": "v29", diff --git a/VERSION b/VERSION index 35c575ccc35a8fce2616d32fdfae4dfb8ca02411..f91cd79924c03ea559bbaed9a6687d0a3cf8e64f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -9445b77 +2de35b7 diff --git a/api/ai_enrich.py b/api/ai_enrich.py index 5b1166fe9506bf6bdd2337ca50691536165dcad8..dbeb6153ad53eaab15f1ff8a38c460c0aa2164e6 100644 --- a/api/ai_enrich.py +++ b/api/ai_enrich.py @@ -314,6 +314,102 @@ def _ask(provider, model, prompt, max_tokens, timeout): return '', None, f"{provider['name']} failed: {type(exc).__name__}" +# =========================================================================== +# W37-T37 (R4 / C7) -- THE GEOCODE FIELD's pure half: turning a record into ONE query string. +# +# ⛔ WHY IT IS HERE AND NOT IN `routes_geo.py`. This module's contract, stated in its own header, +# is that everything in it is a pure function of data it is handed -- which is what lets a gate +# exercise it with no network, no store and no server. Composing an address is exactly that shape. +# `routes_geo.py` owns the part that cannot be pure: the outbound call, the throttle and the cache. +# +# ⭐ WHY IT COMPOSES AT ALL, rather than sending the one column the field names. The field config +# is `{addressField, country?}` -- a single column -- but a customers table stores an address across +# six of them. B MEASURED the fill rates on the real 3,641-row mirror (W37-T11): +# +# street 96.2% · city 96.0% · zip 94.6% · country 95.6% · state 92.0% · street2 5.2% +# street AND city together: 95.9% <- the pair this feature actually needs +# +# Sending `street` alone would ask a global geocoder to place "123 Main Street" with no town, which +# resolves confidently and often wrongly. So the named column LEADS and the conventional siblings +# follow, which is also how a person would write the address down. +# =========================================================================== + +#: The sibling columns appended after the named one, in the order an address is spoken. ⚠ A LIST +#: rather than a formatted string: a template would have to decide what to do about the ~4% of rows +#: missing any one part, and the answer is always "leave it out", never "leave a gap". +GEOCODE_PARTS = ('street2', 'city', 'state', 'zip', 'postal_code', 'country') + +#: ⛔ B's SECOND MEASURED TRAP. Odoo serves `state` as its DISPLAY name, country code included: +#: `New York (US)`, `British Columbia (CA)`. A naive join yields "SCARSDALE, New York (US)", and the +#: parenthetical is noise to a geocoder at best. Stripped here, once, rather than at each caller. +_PAREN_TAIL = re.compile(r'\s*\([^)]*\)\s*$') + + +def geocode_query(row, cfg, seen_parts=None): + """One record plus a geocode field's config -> the query string, or '' when there is nothing. + + ⛔ RETURNS '' RATHER THAN A PARTIAL GUESS when the named column is empty. B measured ~150 of + 3,641 customers with no street at all, and the honest UI for those is "no address on file", not + a request that spends a second of a shared service's budget to place a bare postcode. + + ⚠ `seen_parts=None` RESOLVES `GEOCODE_PARTS` AT CALL TIME, and that is not style. It was + `seen_parts=GEOCODE_PARTS`, which binds the tuple once at import, so the module constant stopped + being the live answer the moment anything wanted to vary it: the gate's own negative control set + `ai_enrich.GEOCODE_PARTS = ()` and the function carried on composing the full address. A default + argument that captures a module constant is a SECOND COPY of it with an earlier timestamp, and + the copy is the one that runs ([[a-constant-two-features-share]]). Caught by that control.""" + if not isinstance(row, dict) or not isinstance(cfg, dict): + return '' + if seen_parts is None: + seen_parts = GEOCODE_PARTS + key = str(cfg.get('addressField') or '').strip() + if not key: + return '' + lead = _clean_part(row.get(key)) + if not lead: + return '' + parts = [lead] + for k in seen_parts: + if k == key: + continue + v = _clean_part(row.get(k)) + # ⚠ The dedupe is not tidiness. `country` is often both the field's own value and already + # present in the street line on an imported record, and "USA, USA" measurably degrades a + # free-form geocoder's confidence. + if v and v.lower() not in {p.lower() for p in parts}: + parts.append(v) + return ', '.join(parts) + + +def _clean_part(v): + """One address component, trimmed and stripped of a trailing country parenthetical.""" + if v is None: + return '' + s = re.sub(r'\s+', ' ', str(v)).strip() + if not s or s.lower() in ('false', 'none', 'null'): + return '' + return _PAREN_TAIL.sub('', s).strip() + + +def geocode_plan(rows, cfg, coord_key=None, overwrite=False): + """Which records a run would actually ask about, and why the rest are being left out. + + ⛔ THIS IS THE R3 GUARD IN ITS ARITHMETIC FORM. It is handed the records a PERSON chose; it + never reads a table. `skipped` is returned rather than swallowed so the surface can say "142 to + look up, 9 have no address, 31 already have coordinates" instead of a bare number.""" + run, skipped = [], {} + for rid, row in (rows or {}).items(): + q = geocode_query(row, cfg) + if not q: + skipped['no_address'] = skipped.get('no_address', 0) + 1 + continue + if coord_key and not overwrite and str((row or {}).get(coord_key) or '').strip(): + skipped['already_located'] = skipped.get('already_located', 0) + 1 + continue + run.append((str(rid), q)) + return {'run': run, 'skipped': skipped} + + def on_change_fields(defn, changed_keys): """Which `ai_enrich` columns in this table want a run because one of their inputs moved. diff --git a/api/ai_review.py b/api/ai_review.py index 35ba8822767b3aea6542587f3e7bd34a3ef81c2d..093c323357f15f25b5301e2db540196f9d7855ee 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -17,15 +17,23 @@ the work. Nothing here can move a card somewhere the review does not already per last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` pins one; `AIOS_AI_MODEL` overrides the model. -⚠ WHY RAW HTTP RATHER THAN THE `anthropic` SDK, stated because it is a deliberate deviation from -the /claude-api skill's default and not an oversight. Three of the four legs are OpenAI-chat-shaped -endpoints with no shared SDK, so a ladder built on the SDK would be one SDK leg beside three -hand-rolled ones — two implementations of the same call, the seam this repo keeps closing. And -`aios-web/api/requirements.txt` is PINNED to what the verify battery proves ([[pin-deps-space- -rebuilds]]): adding a dependency there rebuilds the container, which is a real deploy risk to take -for one leg of an optional feature. `requests` is already a dependency and `harness/analyst.py` -established per-provider raw HTTP as the house pattern. Booked as a DEBT line so the integrator -can overturn it deliberately rather than by drift. +⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH +USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting +the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. +The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the +integrator can overturn it deliberately rather than by drift"*. This is that overturn. + +Its two reasons were real and are both answered rather than waved away: + 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is + not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers + the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no + Anthropic knowledge — it lost some. + 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the + import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the + same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH + manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. +⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that +half of the original reasoning never expired. The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and `stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an @@ -41,9 +49,13 @@ import requests #: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. PROVIDERS = [ + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED + # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent + # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo + # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", "url": "https://api.groq.com/openai/v1/chat/completions", - "model": "llama-3.3-70b-versatile"}, + "model": "openai/gpt-oss-120b"}, {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", "url": "https://api.cerebras.ai/v1/chat/completions", "model": "gpt-oss-120b"}, @@ -161,16 +173,35 @@ def _call_openai(p, model, system, user, timeout): return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body +#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. +#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say +#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. +LAST_ANTHROPIC_TRANSPORT = "" + + def _call_anthropic(p, model, system, user, timeout): - r = requests.post(p["url"], timeout=timeout, - headers={"x-api-key": os.environ[p["env"]].strip(), - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json"}, - json={"model": model, "max_tokens": 300, "system": system, - "messages": [{"role": "user", "content": user}]}) - if r.status_code >= 400: - return "", f"anthropic answered {r.status_code}", None - body = r.json() + """One classification turn on the Messages API, through `providers.anthropic_send`. + + ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which + prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is + absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises + both transports to `(status, body)`, so every line below this call is untouched. + """ + global LAST_ANTHROPIC_TRANSPORT + import providers as _prov # noqa: PLC0415 + # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is + # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. + req = _prov.anthropic_request( + model=model, key=os.environ[p["env"]].strip(), system=system, + messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) + status, body, transport = _prov.anthropic_send(req, timeout=timeout) + LAST_ANTHROPIC_TRANSPORT = transport + if status >= 400 or status == 0: + # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the + # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" + # (which is what this line used to say, and what the owner quoted back at us) cannot recur. + return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None + body = body or {} # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so # reading content[0] before this check turns a refusal into an IndexError inside a run. # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. @@ -444,6 +475,9 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and D asserts it; the two keywords are what D has to pass for the line to be attributable. """ + # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. + # It parses either way; a reader scanning for the declaration does not look inside a `for`. + global LAST_ANTHROPIC_TRANSPORT import usage_ledger # noqa: PLC0415 text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] if not text: @@ -487,13 +521,18 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou # 402), so without this branch the drafter cannot answer at all. # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this # ladder runs. The parameter is the caller's to send, which is why the wire takes it. + # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch + # below reads one thing. The anthropic side gets there through `anthropic_send` (official + # SDK when installed, the identical raw POST when not); the openai side still posts here + # because there is no shared SDK across three different vendors on that wire. + global LAST_ANTHROPIC_TRANSPORT try: if p["shape"] == "anthropic": req = _prov.anthropic_request( model=p["model"], key=os.environ[p["env"]].strip(), system=None, messages=messages, tools=tools, max_tokens=1500, tool_choice="required") - r = requests.post(req["url"], timeout=tmo, - headers=req["headers"], json=req["json"]) + status, body, transport = _prov.anthropic_send(req, timeout=tmo) + LAST_ANTHROPIC_TRANSPORT = transport else: r = requests.post(p["url"], timeout=tmo, headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", @@ -501,19 +540,24 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou json={"model": p["model"], "messages": messages, "tools": tools, "tool_choice": "required", "temperature": 0.1, "max_tokens": 1500}) + status = r.status_code + body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} except Exception as e: # noqa: BLE001 problems.append(f"{p['name']}: {type(e).__name__}") continue - if r.status_code != 200: + if status != 200: # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and # the memo makes the next turn SKIP the empty account instead of paying for it again. - if _prov.is_credit_failure(r.status_code, r.text): + # ⚠ The body is re-serialised for the substring test because the SDK path never had a + # `.text` — that is the one thing this normalisation costs, and it costs nothing else. + detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] + if _prov.is_credit_failure(status, detail): _prov.mark_no_credit(p["name"]) - problems.append(_prov.refusal_sentence(p["name"], r.status_code, r.text)) + problems.append(_prov.refusal_sentence(p["name"], status, detail)) continue try: - body = r.json() + body = body or {} if p["shape"] == "anthropic": _text, args, _refused = _prov.anthropic_read(body) if _refused: diff --git a/api/main.py b/api/main.py index dc3e074067ac710348c9ad0eb79ac66ee3d6ce16..c1b8f579e86345b31a2c74bfd67637dad0f2e781 100644 --- a/api/main.py +++ b/api/main.py @@ -87,6 +87,7 @@ import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's ro import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) +import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router) from core import grid_events # noqa: E402 # D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a @@ -395,6 +396,18 @@ app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent h # Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header # says why that is not a style choice. app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views +# ⭐⭐ WAVE 37 (R14, amendment A11) — THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the +# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the +# artefact the protocol nearly loses, and the first in which the ask that carries it was itself +# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered +# it (D-366). The integrator found it reading the peer mailbox by hand. +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is +# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line — keep it. +# ⛔ R14 put the provider behind a seam precisely so swapping Google in later is a config change: +# tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them +# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). +app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 3659f76145d0bfdeec4ec26fc75c2d8d6878faa0..48eebac0c9c2b04f53e0769a863ec03560421c2b 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -636,8 +636,25 @@ def customer_fields(): "default": True, "pinned": True}, {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", "default": False, "description": "The `res.partner` id. Also this row's id."}, + # ⭐⭐ W37-T11 / D-165 — THE POSTAL ADDRESS, which this table declared nowhere and served + # never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own + # docstring on 2026-08-12 — *"`street`/`street2`/`zip` are not among [the mirror's 16 + # columns] … blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill + # HAS since run (`_sync_state` carries `res_partner.cols.…,street,street2,…,zip`, done + # 2026-08-15), so the half that could not be built now can be. + # ⛔ THIS IS ALSO LANE D'S DEPENDENCY. R4 puts address→coordinates in an enrichment field, + # and a geocoder pointed at a column that does not exist enriches nothing. + # ⚠ `default: False` on the postal lines, `True` on city/state — the same split + # `vendor_fields` made and for the same reason: four address columns on by default push + # the useful ones off the first screen. A locked database still allows fields, so every + # one is a click away in the picker. + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, {"key": "country", "label": "Country", "type": "text", "source": "overlay", "default": False}, {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", @@ -976,8 +993,13 @@ def read_customers(cur, excluded=None): # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. rank_leg = (" OR (p.customer_rank > 0 AND p.active) " if {"customer_rank", "active"} <= have else "") + # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like + # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL + # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the + # live 500 that rule was written after). sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " - f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " + f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, " + f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} " "FROM res_partner p " f"{join}" "WHERE p.id IN (" @@ -988,13 +1010,16 @@ def read_customers(cur, excluded=None): f"{rank_leg}") out = [] for r in cur.execute(sql).fetchall(): - (pid, name, city, state, country, agent, agent_id) = r + (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r out.append({ "_id": str(pid), "customer": str(name or ""), JOIN_KEY: int(pid), + "street": str(street or ""), + "street2": str(street2 or ""), "city": str(city or ""), "state": str(state or ""), + "zip": str(zipc or ""), "country": str(country or ""), "agent": str(agent or ""), AGENT_JOIN_KEY: int(agent_id) if agent_id else "", diff --git a/api/providers.py b/api/providers.py index 462bcc8acdd6509b61cacdb75f82272d853e3954..5425c862136396406714fecaa23eec0d3288ec52 100644 --- a/api/providers.py +++ b/api/providers.py @@ -39,6 +39,12 @@ import time from dataclasses import dataclass, field from typing import Callable +# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in +# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the +# line that stops a container booting. That asymmetry is the whole reason the two are imported +# differently — see `anthropic_sdk()`. +import requests + #: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK #: providers and to estimate a run's spend for the operator. They are not billing truth — the #: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is @@ -352,6 +358,8 @@ LLM_PROVIDERS: dict[str, LlmProvider] = { # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real + # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", wire="anthropic", caps={ @@ -367,24 +375,48 @@ LLM_PROVIDERS: dict[str, LlmProvider] = { "cerebras": LlmProvider( name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", url="https://api.cerebras.ai/v1/chat/completions", + # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the + # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on + # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` + # already knows how to survive (R4: skipped, not tried). model="gpt-oss-120b", wire="openai", caps={ + # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not + # measure this rung: a 402 comes back before any tool is considered, so there is no + # observation to record. Leaving the old wording is the honest answer — upgrading it + # to MEASURED beside its two neighbours would be claiming an account balance as + # evidence about a capability. "llm_tool_calling": Capability(True, 0.0, "DECLARED, not measured: this account's tool-capable " - "model per the wave-32 ladder note"), + "model per the wave-32 ladder note. ⚠ 2026-08-19: the " + "account answers 402, so this stays unmeasured"), "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), "llm_json_mode": Capability(True, 0.0, "response_format json_object"), }), "groq": LlmProvider( name="groq", label="Groq", env="GROQ_API_KEY", + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq + # RETIRED it. Measured, not inferred — a real POST with the live key answered + # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with + # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A + # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the + # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). + # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. + # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while + # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY + # spelled differently and a sweep that "normalises" them breaks one of them. url="https://api.groq.com/openai/v1/chat/completions", - model="llama-3.3-70b-versatile", wire="openai", + model="openai/gpt-oss-120b", wire="openai", caps={ + # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old + # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME + # provider on this wire answers 400 with the call in `failed_generation`; the repo + # never recorded which. Flip this to False the day it is"*. It is recorded now: on + # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed + # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. "llm_tool_calling": Capability(True, 0.0, - "DECLARED, not measured. ⚠ `routes_query._FAILED_GEN` " - "exists because SOME provider on this wire answers 400 " - "with the call in `failed_generation`; the repo never " - "recorded which. Flip this to False the day it is"), + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " + "with parsed arguments; no failed_generation recovery"), "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), "llm_json_mode": Capability(True, 0.0, "response_format json_object"), }), @@ -393,7 +425,11 @@ LLM_PROVIDERS: dict[str, LlmProvider] = { url="https://openrouter.ai/api/v1/chat/completions", model="openai/gpt-4o-mini", wire="openai", caps={ - "llm_tool_calling": Capability(True, 0.0, "DECLARED, not measured"), + # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed + # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and + # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), "llm_json_mode": Capability(True, 0.0, "response_format json_object"), }), @@ -538,16 +574,25 @@ def anthropic_request(*, model, key, system, messages, tools, max_tokens, turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), "content": str(m.get("content") or "")} for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + wire_tools = [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])] body = { "model": str(model), "max_tokens": int(max_tokens), "system": system_text, "messages": turns, - "tools": [{"name": t["function"]["name"], - "description": t["function"]["description"], - "input_schema": t["function"]["parameters"]} for t in (tools or [])], - "tool_choice": {"type": "any" if tool_choice in ("required", "any") else "auto"}, + "tools": wire_tools, } + # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that + # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, + # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no + # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: + # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. + # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and + # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. + if wire_tools: + body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} if effort: body["output_config"] = {"effort": str(effort)} return {"url": LLM_PROVIDERS["anthropic"].url, @@ -557,6 +602,88 @@ def anthropic_request(*, model, key, system, messages, tools, max_tokens, "json": body} +#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. +#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every +#: call, and a failed import is not cheap. `False` says "asked and answered". +_SDK: object = None + + +def anthropic_sdk(): + """The official `anthropic` SDK, or `None` if this deployment does not carry it. + + ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. + `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's + fence this wave, so the pin is another session's edit. A hard `import anthropic` at module + scope would therefore turn a missing line in a manifest into a container that cannot boot at + all: the API imports this module on every request path. + + So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls + back to the raw `requests` POST that has always worked, and says which transport ran. When the + pin lands the SDK path takes over with no further change. + ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body + `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, + `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer + ([[one-question-two-normalizers]]). + """ + global _SDK + if _SDK is None: + try: + import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring + _SDK = _mod + except Exception: + _SDK = False + return _SDK or None + + +def anthropic_send(req, timeout=None, use_sdk=None): + """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. + + ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader + who cannot tell which path ran cannot tell whether the requirements pin reached the container + [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + check keyed to it passes on a stale value written by an earlier check in the same process. + + `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — + the SDK raises where `requests` returns, and normalising that difference here is the only + reason this function exists rather than the caller branching on transport. + + ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports + without assigning to `_SDK`: a test that mutates a module global and then throws leaves every + later check in that process running on the wrong path and passing for the wrong reason. + """ + sdk = None if use_sdk is False else anthropic_sdk() + payload = dict(req.get("json") or {}) + if sdk is None: + r = requests.post(req["url"], headers=req["headers"], json=payload, + timeout=(timeout or 60)) + try: + return r.status_code, (r.json() if r.content else {}), "http" + except Exception: + return r.status_code, {"_text": (r.text or "")[:2000]}, "http" + + key = (req.get("headers") or {}).get("x-api-key") or "" + try: + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + msg = client.messages.create(**payload) + # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same + # wire-shaped dict the raw POST parses out of the response body. + return 200, msg.model_dump(), "sdk" + except Exception as exc: + # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by + # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out + # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. + status = int(getattr(exc, "status_code", 0) or 0) + body = getattr(exc, "body", None) + if not isinstance(body, dict): + body = {"_text": str(exc)[:2000]} + if not status: + # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one + # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. + status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 + return status, body, "sdk" + + def anthropic_read(body): """`(text, tool_input, refusal)` out of a Messages API answer. diff --git a/api/routes_admin.py b/api/routes_admin.py index 7ee38d40363f882cd229ca9c8922db6cdbd4d6a9..94a337ed21bb47bcd146ad088378fa0109c66d11 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -1,1187 +1,1234 @@ -"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). - -These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` -modals, over the SAME `core/users` account store — so both front-ends administer one set of -accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. - -ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit -dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record -is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. - -⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` -has no caller anywhere in the shipped app — module grants are only settable by editing the store -JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both -asserted in `verify_api.py` section A: - - * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin - clearing every checkbox to lock an account down would grant it everything. - * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to - `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose - whole point is strict isolation. - -Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in -`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the -WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both -read semantics are untouched — the seam is write-strict / read-unchanged. - -⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` -one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no -accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". -""" -import os -import re - -from fastapi import APIRouter, Body, Depends, Response - -import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag -import core.registry as registry -import core.store as store - -from deps import Session, err, perms, require_session, users -from routes_auth import _public_user - -router = APIRouter(prefix="/api/v1") - -#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under -#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no -#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. -_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") -#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog -#: enforces nothing; this does, and the two are allowed to differ because only one of them is -#: reachable from the internet. -_MIN_PW = 8 -_ROLES = ("user", "admin") - - -def admin_gate(session: Session = Depends(require_session)) -> Session: - """401 without a session, 403 without the admin role. Role, not a module grant — mirroring - `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - return session - - -def _require_store(): - """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" - try: - ok = store.available() - except Exception: - ok = False - if not ok: - raise err(503, "store_unavailable", - "the tenant store is unavailable — accounts cannot be read or changed") - - -def _registry(): - _require_store() - try: - reg = users.registry() or {} - except Exception: - raise err(503, "store_unavailable", "the tenant store is unavailable") - return reg - - -def _view(uname, rec, governed=None, st=None, surface_keys=None): - """What an administrator may see about an account. NEVER `salt` or `hash` — this function is - the only projection these routes use, so there is one place that can leak and it does not. - - ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both, - resolved once for the whole request; the single-record routes pass neither, because their - caller fetches the editor payload separately and `GET /perms` is the authority there. - """ - return {"username": uname, - "name": rec.get("name") or uname, - "role": rec.get("role", "user"), - "bus": rec.get("bus", "all"), - "bu_labels": perms.allowed_bu_labels(rec), - "modules": rec.get("modules", "all"), - "agent": rec.get("agent") or None, - "email": rec.get("email") or None, - "tenant": _tenant_of(rec), - "active": bool(rec.get("active", True)), - # The session-revocation handle. An admin needs it: it is the only visible evidence - # that "sign them out everywhere" actually happened. - "epoch": int(rec.get("epoch") or 0), - # S3 ask 4 — one sentence for the roster's Access column, so the list does not need - # a /perms round trip per row to fill one cell. Computed from the same record the - # editor will open, so the two cannot disagree. - "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st, - surface_keys=surface_keys), - "perms_v": int(rec.get("perms_v") or 0)} - - -def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): - """One sentence describing what this account may reach. Deliberately says LESS than the - editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters - would be wrong the moment a filter got interesting. - - ⛔⛔ W36-T22 — IT MUST COUNT WHAT THE EDITOR WOULD SHOW, NOT WHAT IS STORED, AND THE TWO STOPPED - AGREEING THE MOMENT `get_perms` LEARNED TO DEFAULT FROM `may_read`. This counted STORED ENTRIES - only, which WAS the same answer before this wave: a migrated account had an entry for every - governable database, because the editor wrote one per topic on every save and no `ut_*` entry - could exist at all. Now a `ut_*` key legitimately has NO entry and is still OPEN, so the roster - would say *"1 module"* about an account the editor shows reaching twelve. ⛔ The roster is the - screen an administrator reads FIRST, and two screens disagreeing about one account is the - defect this wave exists to remove, not a rounding difference. - - ⚠ `governed`/`st` ARE PASSED IN RATHER THAN DERIVED HERE, and that is a cost decision. This - runs once PER ROW of the roster; asking `may_read` per (account x database) without a lend - would be a whole-document copy per question — D-213's shape, one route over. The caller - resolves the tenant's list and ONE lend for the whole request. A caller that passes neither - keeps the stored-entry reading, which is right for the single-record routes: their client - fetches `GET /perms` separately and that payload is the authority. - """ - import core.perm_scope as perm_scope - - if perms.is_admin(rec): - # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if - # they applied is the one misreading of that clause that could actually hurt. - return "Everything (admin)" - if not perm_scope.is_migrated(rec): - mods = perms.allowed_modules(rec) - return "All modules (legacy rules)" if mods is None else \ - f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" - entries = (rec.get("perms") or {}) - if governed: - principal = users._public(uname, rec) if uname else rec - # ⛔ PER KIND, MIRRORING `get_perms`' OWN DEFAULTS — the first draft asked - # `nav_may_open` for EVERY key and that was a second idea of the count: for a migrated - # record with no topic entry the editor's default is `may_read` (DENY), so a blanket - # admission read would make the roster say "3 modules" about an account whose editor - # shows one box ticked. The roster's whole contract (leg 6 of the T22 section) is that - # it counts what the editor would show; a SURFACE key defaults from admission there, - # everything else from `may_read`, so this does exactly the same, key by key. - opened = [k for k in governed - if (perm_scope.nav_may_open(principal, k, st=st) - if k in (surface_keys or ()) else - perm_scope.may_read(principal, k, st=st))] - else: - opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] - restricted = sum(1 for k in opened - if isinstance(entries.get(k), dict) - and (entries[k].get("filter") or entries[k].get("hiddenFields"))) - if not opened: - return "No modules" - base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" - return f"{base}, {restricted} restricted" if restricted else base - - -# ── request validation: every rule below is fail-closed ────────────────────────────────────────── -def _clean_username(v): - uname = str(v or "").strip().lower() - if not _UNAME_RE.match(uname): - raise err(400, "bad_username", - "a username is 2-32 characters: lowercase letters, digits, dot, dash or " - "underscore, starting with a letter or digit") - return uname - - -def _clean_password(v): - pw = str(v or "") - if len(pw) < _MIN_PW: - raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") - return pw - - -def _clean_role(v): - role = str(v or "").strip().lower() - if role not in _ROLES: - raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") - return role - - -def _clean_bus(v): - """'all', or a non-empty list of KNOWN business-unit ids. - - Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no - recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_bus", - "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " - "would read as unrestricted)") - out = [] - for b in v: - try: - b = int(b) - except (TypeError, ValueError): - raise err(400, "bad_bus", "a business-unit id must be a number") - if b not in users.BU_LABELS: - raise err(400, "bad_bus", - f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " - "unrecognised id would widen access to every BU") - out.append(b) - return sorted(set(out)) - - -def _clean_modules(v): - """'all', or a non-empty list of KNOWN module keys. - - `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is - refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would - silently lock the account out of the page they meant to grant. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_modules", - "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " - "reads as UNRESTRICTED, which is the opposite of locking an account down") - known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) - out, bad = [], [] - for k in v: - k = str(k or "").strip() - (out if k in known else bad).append(k) - if bad: - raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") - return sorted(set(out)) - - -def _tenant_of(rec): - """The account's company, with the pre-wave default. ONE spelling of the default, used by - every admin route — two spellings is how a tenant wall grows a gap.""" - return str((rec or {}).get("tenant") or "royal-imports").strip().lower() - - -# ── the routes ─────────────────────────────────────────────────────────────────────────────────── -@router.get("/admin/users") -def list_users(session: Session = Depends(admin_gate)): - """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global - control-plane bucket, so without this filter a Nurilab admin would read Royal's whole - roster — names, emails, agents — from one URL.""" - reg = _registry() - mine = _tenant_of(session.user) - # ⭐ W36-T22 — the tenant's governable list and ONE lend, resolved once for the whole roster so - # `_access_summary` can agree with the editor without paying a document copy per question. - import core.user_tables as _ut - try: - _rows = _perm_modules(session, surfaces=True) - governed = [m["key"] for m in _rows] - surface_keys = {m["key"] for m in _rows if m.get("surface")} - lent = _ut.lend_defs(session.runtime) - except Exception: # noqa: BLE001 - # A store that cannot list the tenant's databases must not take the roster down. The - # summary degrades to the stored-entry reading, which is what it always was. - governed, lent, surface_keys = None, None, None - return {"users": [_view(u, reg[u], governed=governed, st=lent, surface_keys=surface_keys) - for u in sorted(reg) - if _tenant_of(reg[u]) == mine]} - - -@router.post("/admin/users", status_code=201) -def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): - """Create an account. **409 if the username already exists — `PATCH` is the update path.** - - ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh - record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and - RESURRECT every cookie minted before its last password change. `core/users.py` now - preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / - update a user" calls the same function), but this route still refuses: an upsert that silently - replaces an account's password, role and BU access because someone reused a username is not an - update anybody asked for. - """ - body = body or {} - _require_store() - uname = _clean_username(body.get("username")) - pw = _clean_password(body.get("password")) - role = _clean_role(body.get("role") or "user") - bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") - modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") - name = str(body.get("name") or "").strip() or uname - agent = str(body.get("agent") or "").strip() or None - email = str(body.get("email") or "").strip() or None - - if uname in _registry(): - raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") - # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — - # the tenant is stamped from the session, not taken from the body. - users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, - agent=agent, email=email, tenant=_tenant_of(session.user)) - reg = _registry() - if uname not in reg: - # The store accepted the write and does not have it. Reporting 201 here would be the - # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. - raise err(503, "store_unavailable", "the account was not saved — try again") - return {"user": _view(uname, reg[uname])} - - -@router.patch("/admin/users/{username}") -def update_user(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Change access on an existing account. Only the keys PRESENT in the body change. - - ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every - request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very - next call — bumping the epoch would only add a forced re-login on top. `active: false` is the - exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its - session rather than keep working until the cookie expires. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if not body: - raise err(400, "empty_patch", "no fields to update") - - unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) - if unknown: - # A silently-ignored field is how a UI ends up believing it saved something it did not. - raise err(400, "unknown_fields", f"cannot update: {unknown}") - - is_self = uname == session.uname - role = _clean_role(body["role"]) if "role" in body else None - active = bool(body["active"]) if "active" in body else None - # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is - # not the thing that keeps the product reachable — it just stops an administrator removing - # their own access with a toggle and having to go find the master password. - if is_self and role is not None and role != "admin": - raise err(400, "self_demote", - "you cannot remove your own administrator role — ask another admin") - if is_self and active is False: - raise err(400, "self_deactivate", "you cannot deactivate your own account") - - kw = {} - if role is not None: - kw["role"] = role - if "name" in body: - kw["name"] = str(body["name"] or "").strip() or uname - if "bus" in body: - kw["bus"] = _clean_bus(body["bus"]) - if "modules" in body: - kw["modules"] = _clean_modules(body["modules"]) - if "agent" in body: - kw["agent"] = str(body["agent"] or "").strip() # '' clears the link - if "email" in body: - kw["email"] = str(body["email"] or "").strip() - if kw: - users.set_access(uname, **kw) - if active is not None and active != bool(reg[uname].get("active", True)): - users.set_active(uname, active) # bumps the epoch: kills live sessions - - fresh = _registry() - return {"user": _view(uname, fresh[uname])} - - -@router.post("/admin/users/{username}/password", status_code=204) -def set_password(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump - happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two - can never disagree. - - 204 with no body: there is nothing to say that the caller did not already know, and echoing the - account back would invite a client to diff a response for a password change. - """ - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - pw = _clean_password((body or {}).get("password")) - before = int(reg[uname].get("epoch") or 0) - users.set_password(uname, pw) - after = int((_registry().get(uname) or {}).get("epoch") or 0) - if after <= before: - # The whole value of this route is the revocation. If the epoch did not move, the sessions - # were not revoked, and answering 204 would say they were. - raise err(503, "store_unavailable", - "the password change did not persist — outstanding sessions were NOT revoked") - return Response(status_code=204) - - -# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── -#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema -#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list -#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` -#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), -#: so a migrated account would be locked out of Product with no control anywhere to grant it: -#: fail-closed, but unadministrable. The editor grows its section with zero client edits -#: (S3 built it off this route's `modules` + `fields_by_module`). -#: -#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be -#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple -#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid -#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` -#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, -#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` -#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s -#: `section_tenant_derived` NC does exactly that and reds. -#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from -#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, -#: non-group_only registry row that declares a `topic`. What stays hand-written is -#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` -#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this -#: editor at all. The intersection is the honest answer to "what can this module build pickers for". -#: -#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads -#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the -#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise -#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red -#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives -#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. -_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) - - -def _governable_catalogue(): - """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. - - ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is - tenant-blind by design. Two filters in two files answering one question is how the perms route - and `routes_nav` came apart in the first place. - """ - import core.registry as registry - return [{"key": v["key"], "label": v["label"]} - for v in registry.governable_modules().values() - if v.get("key") in _FIELD_PROVIDER_KEYS] - - -_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) - - -def _perm_modules(session, surfaces=False): - """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. - - Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned - module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and - the tenant's own `ut_*` databases read through the DEFINITIONS projection. - - ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE - resolver for "which databases may this account see", it is `may_open`-filtered, and it reads - the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. - The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver - instead of assuming that is what keeps this from becoming the SECOND idea of who may see a - table (`user_tables.may_open`'s own docstring is the record of the first time that happened). - """ - import core.perms as _perms - import core.user_tables as user_tables - try: - ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, - st=session.runtime) - except Exception: - # A store that cannot list the tenant's databases must not take the whole editor down — - # the topic half is still administrable, and an empty `ut_*` half is visibly empty. - ut = () - cat = _governable_catalogue() - skeys = set() - if surfaces: - # ⭐⭐ OWNER RULING 2026-08-18 — the Assistant and Agents surfaces are - # governable ACCESS toggles in the ACCOUNT editor. `surfaces` is caller-explicit and - # defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor - # (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE - # (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle - # there would be a stored rule nothing applies, the exact class C2 deleted. - surf = _perms.governable_surface_modules() - skeys = {s["key"] for s in surf} - cat = cat + surf - rows = _perms.tenant_governable_modules(session.runtime, cat, ut_entries=ut) - # The marker survives OUTSIDE `tenant_governable_modules`, which builds bare {key,label} - # rows: the client needs it only to choose the schemaless help copy, and the server needs - # it only to pick the admission default in `get_perms`. - return [dict(r, surface=True) if r["key"] in skeys else r for r in rows] - - -#: ⛔ `_enforced_keys` IS DELETED (W36-T22 / CONTRACT C2). It asked `perms.enforced_module_keys` -#: which subset of the listed databases a wall could actually be stored against, and after R6 the -#: answer is "all of them" — a question with one possible answer is not a question. Both it and the -#: `perms` helper behind it are gone rather than made to return everything, because a predicate -#: that ignores its argument is the always-true flag C2 deletes, one indirection along. -#: What callers use instead is `{m["key"] for m in _perm_modules(session)}` — the SAME list the -#: editor renders, so "shown" and "storable" cannot drift apart again. - - -def _module_fields(key, session=None): - """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, - so an admin filters on exactly what the user sees. - - ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing - what to hide, so hiding the choices would make a field unhideable the moment it was hidden - once for somebody. The route is admin-gated; the projection a MEMBER receives is the one - `perm_scope.visible_fields` narrows. - - ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the - editor the customer schema for it would let an admin build a wall out of columns that - module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, - a filter naming a field the product rows lack would DENY every row via `permits()`. - The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader - is served fewer columns, but the ADMIN is choosing what may ever be hidden. - - ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read - `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that - was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being - a two-element literal, a `ut_*` database or a third topic would have been handed customer - columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the - stored wall would name columns that module does not have — which `permits()` resolves by - DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: - a topic without a schema arm is a missing arm, never a customer grid in disguise. - - ⭐⭐ W36-T22 / R6 — AND `ut_*` NOW HAS AN ARM, WHICH IS THE HALF THAT MAKES THE TOGGLE REAL. - Owner item 11: *"how come the database is only toggleable for Odoo customers and Odoo - products … EVERY database should be able to be toggleable by admin."* A `ut_*` database's - field contract is its OWN stored definition, so the picker offers exactly the columns that - database has. ⛔ ONE SOURCE, NOT TWO: the picker (`get_perms.fields_by_module`) and the - validator (`_clean_perms`) both call this, so a column the picker offers can never be a - column the validator rejects. Getting that wrong is worse than an error message — - `permits()` DENIES on a predicate it cannot answer, so a mismatched wall is a - deny-everything trap wearing a saved-successfully toast. - - ⚠ `session` IS REQUIRED FOR A `ut_*` KEY and unused for a topic: a stored definition is - per-tenant data and there is no tenant-blind way to read one. A caller that omits it is - refused LOUDLY rather than handed another tenant's schema — the same rule the `else` - fall-through above was deleted for. - """ - import aios_grid - key = str(key or "") - if key in {s["key"] for s in perms.governable_surface_modules()}: - # ⭐ A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest - # answer rather than the wave-26 empty-option trap: the editor's schemaless rendering - # is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any - # filter or hiddenFields validated against this empty vocabulary. An unknown key - # still RAISES below — this arm admits exactly the derived surface list. - return [] - if key.startswith("ut_"): - if session is None: - raise err(400, "ungoverned_module", - f"{key!r} is a tenant database, so its columns can only be read for a " - f"known tenant — this caller passed no session") - import core.user_tables as user_tables - defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime)) - if not defn: - raise err(400, "ungoverned_module", - f"{key!r} is not a database in this workspace") - src = defn.get("fields") or [] - else: - providers = {"customer_data": lambda: aios_grid.FIELDS, - "product_data": aios_grid.product_fields} - provider = providers.get(key) - if provider is None: - # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative - # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is - # truthy. - raise err(400, "ungoverned_module", - f"{key!r} has no permission-editor field schema. The editor governs " - f"{sorted(providers)} and this tenant's own databases") - src = provider() - known = _server_side_vocabularies() - out = [] - for f in src: - if not isinstance(f, dict) or not f.get("key"): - continue - opts = f.get("options") - if not isinstance(opts, list) or not opts: - # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See - # `_server_side_vocabularies`. - opts = known.get(f["key"]) - out.append({"key": f["key"], "label": f.get("label") or f["key"], - "type": f.get("type") or "text", - **({"options": list(opts)} if isinstance(opts, list) and opts else {}), - **({"pinned": True} if f.get("pinned") else {})}) - return out - - -def _server_side_vocabularies(): - """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, - but which the grid contract does not declare. - - ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid - resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the - field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so - every column that leans on the second branch renders `Select…` with nothing in it — the - identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix - belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied - list as authoritative *including when it is empty* ("this column has no values" is an answer), - which is correct and must not be weakened. So the caller stops handing it nothing. - - ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, - the same constant `_bucket()` mints from — a copy here would be a second vocabulary that - drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy - for the same reason `aios_grid`'s is: this route should not pull the analytics stack at - module load. - - ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than - let them look handled: - · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no - closed list to declare, and discovering it would mean a pool read on an admin route. - · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E - answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which - is the right home (the grid picks a declared list up for free) and is why this function - never needed an arm for it. Recorded because the ask, not a guess, is what settled it. - """ - try: - import modules.inventory as inventory - labels = list(inventory.COVERAGE_LABELS or ()) - except Exception: - # A missing analytics import must not take the permission editor down; the dropdown - # degrades to today's empty one rather than the page to a 500. - return {} - return {"stock_bucket": labels} if labels else {} - - -def _clean_perms(v, governed_keys=None, session=None): - """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. - - ⭐ WAVE 33 (W33-T33) — `governed_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It - used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 - flagged: an admin of a tenant provisioned for neither grid module could still store a wall - for both. It now comes from `_perm_modules(session)`, i.e. from the tenant's own catalogue. - The default is the topic catalogue itself so a test or a future caller cannot silently widen - it. - - ⭐⭐ W36-T22 / CONTRACT C2 — **THE `enforced`/`listed` SPLIT IS GONE, AND SO IS - `unenforced_module`.** This function took TWO key sets and refused anything in the gap - between them, with a message that ended *"Share the database instead, or arm `perm_scope` - over `ut_*` first."* W36-T21 armed it. There is no gap left: every database the editor lists - is a database whose wall `routes_tables` applies on every read, so a wall stored against one - is enforced by the same code that enforces `customer_data`'s. **Two sets collapse into one**, - which is the shape C2 asks for — not a second set that is always equal to the first. - - ⚠ `session` IS THREADED SO `_module_fields` CAN READ A `ut_*` SCHEMA. Without it the - validator would have no field contract for the databases this ticket just made governable, - and "no contract" resolves to "every `hiddenFields` entry is unknown" — a 400 on the admin's - own screen. - - ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately - drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION - filter is the opposite situation — an admin types "agent is Tara", a leaf is silently - dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole - book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: - the filter is re-validated with `clean_filter_tree` and the result must come back with the - SAME leaf count it went in with. - - Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` - validates the NODE LIST alone and never sees the root conjunction, so the two are checked - separately and a bare node list is refused rather than silently read as `and`). - """ - import aios_grid - - if v is None: - return None - if not isinstance(v, dict): - raise err(400, "bad_perms", "perms must be an object keyed by module") - governed = set(governed_keys) if governed_keys is not None else set(_PERM_MODULES) - unknown = sorted(set(v) - governed) - if unknown: - raise err(400, "bad_perms", - f"unknown module keys: {unknown} — this tenant's permission editor governs " - f"{sorted(governed)}") - out = {} - for key, raw in v.items(): - if not isinstance(raw, dict): - raise err(400, "bad_perms", f"{key}: each entry must be an object") - valid_keys = {f["key"] for f in _module_fields(key, session=session)} - hidden = raw.get("hiddenFields") or [] - if not isinstance(hidden, list): - raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") - bad = sorted({str(h) for h in hidden} - valid_keys) - if bad: - # A hiddenFields entry naming nothing hides nothing — and reads as a restriction - # that is not there. - raise err(400, "bad_perms", - f"{key}: hiddenFields names unknown fields {bad}") - tree = raw.get("filter") - clean_tree = None - if tree not in (None, {}, []): - if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): - raise err(400, "bad_filter", - f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " - f"the root conjunction, and an 'or' wall read as 'and' restricts " - f"nothing it was meant to") - conj = tree.get("conj", "and") - if conj not in ("and", "or"): - raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") - nodes = tree["nodes"] - cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) - if _leaf_count(cleaned) != _leaf_count(nodes): - raise err(400, "bad_filter", - f"{key}: the filter contains conditions this module cannot evaluate " - f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " - f"are per-user and cannot be a permanent rule). Refused rather than " - f"saved with the bad conditions silently removed, which would store a " - f"weaker wall than the one on screen.") - clean_tree = {"conj": conj, "nodes": cleaned} - out[key] = {"access": bool(raw.get("access", True)), - "filter": clean_tree, - "hiddenFields": sorted({str(h) for h in hidden})} - _refuse_unshapeable_bu(out) - return out - - -def _refuse_unshapeable_bu(perms_out): - """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. - - Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes - that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which - is what `perm_migrate` emits and what the condition builder produces for "is any of". An - admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a - `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through - `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries - Fisch+Royal numbers. The row list looks right. The revenue is another BU's. - - That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just - shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with - nothing anomalous about them. - - So it is refused at the WRITE, where a person is present to fix it. The discriminator is - narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like - `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and - consolidated values are the correct answer for it, so it is not caught here (`derive` also - refuses it) and must not be. - """ - import core.perm_scope as perm_scope - - for key, e in perms_out.items(): - tree = e.get("filter") - if not tree or not _mentions_dba(tree.get("nodes")): - continue - probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, - "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} - team_id, _ = perm_scope.derive_pool_scope(probe, key) - if team_id is None and _confines_to_one_bu(tree): - raise err(400, "bu_condition_shape", - f"{key}: this filter restricts the business unit in a way the data layer " - f"cannot push into the query, so the rows would be correct while the " - f"revenue figures on them stayed consolidated across both units. Express " - f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " - f"or an 'any of' over the brands — and keep any other rules alongside it.") - - -def _guard_bu_widening(rec, cleaned, confirmed): - """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. - - `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the - first save the filter IS the whole wall and the admin owns it. But the first save also - CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted - rather than the one the server returned will, on its next save, PUT a payload with no BU - condition — and a whole-record replace deletes it. Measured, not theorised: the same client - payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both - business units. - - So a save that REMOVES the brand confinement is refused unless it says it means to. Widening - BU access is a legitimate thing for an administrator to do — it is just never a thing to do - by accident, and the difference between the two is one explicit flag. - - ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU - restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the - admin edits it). The right shape is "you may widen, say so". - """ - import core.perm_scope as perm_scope - - if confirmed: - return - stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} - for key, new_entry in cleaned.items(): - if not new_entry.get("access", True): - # Revoking the module entirely is the OPPOSITE of widening, and the filter on a - # denied entry governs nothing. Guarding it would make "lock this account out" - # require a confirm_widen flag, which reads as nonsense to whoever hits it. - continue - old = stored.get(key) - if not isinstance(old, dict): - continue # nothing stored yet: the fold already handled it - old_tree, new_tree = old.get("filter"), new_entry.get("filter") - if not old_tree or not _confines_to_one_bu(old_tree): - continue # was not confined -> this save cannot widen it - if new_tree and _confines_to_one_bu(new_tree): - continue # still confined -> not a widening - raise err(400, "bu_widening", - f"{key}: this would REMOVE the business-unit restriction and give the account " - f"both units. If that is intended, resend with confirm_widen: true. If it is " - f"not, reload the account's permissions first — saving a filter loaded before " - f"the last change drops the conditions it did not know about.") - # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees - # the connection to derive_pool_scope, which is what the removed condition was feeding. - _ = perm_scope - - -def _mentions_dba(nodes): - for n in nodes or (): - if not isinstance(n, dict): - continue - if isinstance(n.get("children"), list): - if _mentions_dba(n["children"]): - return True - elif n.get("colId") == "dba": - return True - return False - - -def _confines_to_one_bu(tree): - """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? - - ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard - refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a - high-balance Royal customer satisfies it — so consolidated values are the correct answer and - refusing it would be a false positive. My first attempt evaluated synthetic rows carrying - only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact - wall into a "confinement" it is not. - - So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the - reader satisfies everything else; can they still see both brands?" If yes, the wall does not - confine and the pool is correctly consolidated. If no, the brand structure is doing the - confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. - """ - def admits(node, brand): - if isinstance(node.get("children"), list): - kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] - if not kids: - return True - return any(kids) if node.get("conj") == "or" else all(kids) - if node.get("colId") != "dba": - return True # every other condition: assume satisfied - want = str(node.get("value") or "").strip().lower() - op = node.get("op") - if op == "eq": - return brand.lower() == want - if op == "neq": - return brand.lower() != want - if op == "contains": - return want in brand.lower() - if op == "doesNotContain": - return want not in brand.lower() - if op == "isEmpty": - return False # a probe brand is never blank - if op == "isNotEmpty": - return True - return True # an op that cannot judge a brand does not confine - - nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") - root = {"conj": conj, "children": nodes} - admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} - if not admitted: - return False # admits nothing at all — a different problem - # 'Both' belongs to either unit, so it cannot by itself widen the answer. - return not ({"Fisch", "Royal"} <= admitted) - - -def _leaf_count(nodes): - n = 0 - for node in nodes or (): - if isinstance(node, dict) and isinstance(node.get("children"), list): - n += _leaf_count(node["children"]) - else: - n += 1 - return n - - -def _fold_legacy_scope(rec, cleaned): - """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. - - Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` - fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write - whose filter carries no BU condition silently promotes a Royal-only account to BOTH business - units: the wall that used to come from `bus` is gone and nothing replaced it. - - Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check - passed, because the model was doing exactly what it was told. The account simply received - the other BU's customers on the wire. - - So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as - `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a - perms block. From then on the admin sees those leaves in the editor (GET returns the folded - filter) and can change them like any other condition — which is R1's whole point, and why - this is a one-time fold rather than a permanent floor AND-ed on every write. - - Idempotent by construction: it runs only while the record is un-migrated, and the write that - calls it is the write that migrates it. - - ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE - NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` - has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent - filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into - the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for - a `bus:[5]` account the moment an admin pressed Save. - - Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against - `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a - cross-topic leaf could come through, which is why it is the only place that needs the rule. - - ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the - BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU - is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). - The pushdown falls back to the record's own `bus` for exactly the modules whose field - vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves - ship together or the second one is a leak. - """ - import core.perm_migrate as perm_migrate - import core.perm_scope as perm_scope - - if perm_scope.is_migrated(rec): - return cleaned - legacy = perm_migrate.filter_for(rec) - if not legacy: - return cleaned # nothing to preserve: the account was unrestricted - out = {} - for key, e in cleaned.items(): - legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) - tree = e.get("filter") - if not legacy_here: - # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than - # storing a wall it will read as "deny every row". - out[key] = e - continue - if not tree or not tree.get("nodes"): - folded = legacy_here - else: - # AND the two roots together. The submitted tree keeps its own conjunction by being - # nested as a GROUP — flattening an `or` tree into an `and` root would turn the - # admin's "A or B" into "A and B", which is a different and much narrower wall. - folded = {"conj": "and", - "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), - "children": list(tree["nodes"])}]} - out[key] = dict(e, filter=folded) - return out - - -def _prune_to_module(tree, valid_keys): - """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. - - Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies - every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the - reason its own docstring gives (a silently-weakened wall reads as "Saved"). - - ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than - the admin's intent and half of an `and` group is a wider one; neither is the thing that was - written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps - the surviving conditions ones somebody actually declared. - """ - if not isinstance(tree, dict): - return None - - def keep(node): - if not isinstance(node, dict): - return None - kids = node.get("children") - if isinstance(kids, list): - surviving = [keep(k) for k in kids] - if not kids or any(s is None for s in surviving): - return None - return dict(node, children=surviving) - return node if node.get("colId") in valid_keys else None - - nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] - return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None - - -@router.get("/admin/users/{username}/perms") -def get_perms(username: str, session: Session = Depends(admin_gate)): - """This account's permission block plus the field schema the editor needs to render it.""" - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - import core.perm_scope as perm_scope - - rec = reg[uname] - stored = rec.get("perms") or {} - # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. - # The PUT is a whole-record replace over this same module list, so a key declared here and - # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The - # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the - # access its legacy grant currently gives rather than a guess in either direction. - # - # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything - # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a - # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product - # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` - # has applied since wave 18) and merges the tenant's own `ut_*` databases. - modules = _perm_modules(session, surfaces=True) - # ⭐⭐ W36-T22 / C2 — EVERY listed database, not an `enforced` subset of them. See - # `perms.tenant_governable_modules` for why the flag is deleted rather than defaulted. - governed = [m["key"] for m in modules] - perms_out = {} - for k in governed: - e = stored.get(k) - # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN - # OUTAGE. `may_access` reads migrated-and-undeclared as DENY. That is right for a registry - # topic and catastrophic for a `ut_*` key, because no `ut_*` entry was STORABLE before this - # wave — so every migrated account carries none, and this route would default every one of - # them to `access: false`. - # - # ⛔ AND IT WOULD NOT HAVE STAYED A DISPLAY BUG FOR LONG. C2 also deleted the filters that - # kept `ut_*` keys OUT of the PUT body, so the editor now sends every declared module: an - # administrator opening this page, changing one unrelated filter and pressing Save would - # write an EXPLICIT `{access: false}` for all ten keychain databases — at which point - # `may_read`'s deny-only overlay fires and revokes them for real. One click, silent, - # permanent. Defaulting from the SAME evaluator the read door uses is what makes the page - # show what the user can actually open, so a save of an untouched payload is a no-op. - # ⚠ `_public(uname, rec)`, NOT the raw record. A stored record is keyed BY username in - # `users.json` and does not carry one INSIDE it, so `may_read` would hand `may_open` a - # `None` viewer and get a fail-closed False — the very outage this line exists to prevent, - # arriving through the fix for it. The same trap cost a gate double an hour earlier today. - # ⛔⛔ A SURFACE KEY DEFAULTS FROM `nav_may_open`, NOT `may_read`, AND THE - # DIFFERENCE IS A MASS REVOCATION. `may_read` on a non-`ut_` key falls through to - # `may_access`, which reads migrated-and-undeclared as DENY — correct for a topic - # (every save writes topic entries) and false for a surface, because NO surface entry - # was storable before 2026-08-18, so every migrated account carries none. The editor - # would paint Assistant unchecked for an account that opens it every day, and the next - # save of ANY unrelated change would write the explicit deny for real. `nav_may_open` - # is the evaluator the nav and the route gate actually ask, legacy fallback included, - # so the box shows what the account can reach and an untouched save stays a no-op. - _default = (perm_scope.nav_may_open(users._public(uname, rec), k) - if any(m.get("surface") and m["key"] == k for m in modules) else - perm_scope.may_read(users._public(uname, rec), k, st=session.runtime)) - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(_default), "filter": None, "hiddenFields": []} - return {"username": uname, - "perms": perms_out, - # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a - # migrated record and LEGACY on an un-migrated one: opposite meanings for identical - # JSON, so the client cannot render honestly without knowing which world it is in. - "perms_v": int(rec.get("perms_v") or 0), - # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can - # say "Everything (admin)" instead of rendering stored rules that do not apply. - "is_admin": perms.is_admin(rec), - # ⭐⭐ W36-T22 / C2 — `{key, label}` per row and NO `enforced`. It used to ride here - # so the editor could paint an apology ("Not set here") for a database whose wall was - # never applied; W36-T21 armed every one of them, so the flag would now be constantly - # true and the apology would be a lie with a green gate behind it. The LABEL comes - # from the registry / the table's own definition. - "modules": modules, - # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav - # schema envelope; the editor needs the field list and nothing else, and one shape - # beats two readings of a sentence. - # ⭐ EVERY governed database now carries its fields, `ut_*` included — that is owner - # item 11's *"EVERY database should be able to be toggleable by admin"*, and it is - # the SAME resolver `_clean_perms` validates against, so the picker cannot offer a - # column the validator will reject. - "fields_by_module": {k: _module_fields(k, session=session) for k in governed}} - - -@router.put("/admin/users/{username}/perms") -def put_perms(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this account's permission block wholesale. - - WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot - express a deletion without a second vocabulary for it. - - ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: - `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the - target's LIVE session on its very next call. Bumping would only add a forced re-login on top - of a change that has already taken effect. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - - # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses - # perm_scope entirely, so this cannot brick them today — but it stops an administrator - # writing a wall for themselves that would bite the moment their role changed. - if uname == session.uname: - raise err(400, "self_perms", - "you cannot set permissions on your own account — ask another admin") - - _mods = _perm_modules(session, surfaces=True) - # ⭐⭐ W36-T22 / C2 — ONE key set. It was two ("listed" and "enforced") with a refusal in the - # gap; W36-T21 closed the gap, so every database the editor shows is one whose wall the table - # routes apply. `session` rides so a `ut_*` schema can be read for THIS tenant. - cleaned = _clean_perms(body.get("perms"), - governed_keys={m["key"] for m in _mods}, session=session) or {} - cleaned = _fold_legacy_scope(reg[uname], cleaned) - _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) - users.set_access(uname, perms=cleaned) - fresh = _registry() - rec = fresh.get(uname) or {} - if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: - # The store took the write and did not record it. A 200 here would tell an administrator - # the wall is up when it is not — the one direction this must never fail in. - raise err(503, "store_unavailable", - "the permissions were not saved — the account is UNCHANGED") - return {"username": uname, "perms": rec.get("perms") or {}, - "perms_v": int(rec.get("perms_v") or 0)} - - -def _store_binding(session): - """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. - - ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 - rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset - repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path - that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only - tenant #0. Reporting the process default here would show "staging store, all fine" to exactly - the tenants that were not fine. - - ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it - could not describe the store is worse than one that says it does not know. - """ - try: - import core.data_binding as data_binding # noqa: PLC0415 - import core.store as store # noqa: PLC0415 - bound = getattr(session.runtime, "store_handle", None) - repo = getattr(bound, "repo", None) or store.REPO - return data_binding.describe(repo) - except Exception as e: # noqa: BLE001 - return {"deployment": "unknown", "production": False, "repo": "", - "writable": False, "refusal": f"could not resolve the store binding: {e}", - "localOverride": False, "allowlist": [], "observed": {}} - - -@router.get("/settings") -def settings(session: Session = Depends(require_session)): - """The session's OWN settings — any authenticated user, not admin-only. - - `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a - client may know about itself (never a hash, never the epoch). `scope` restates it as the two - things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others - would answer a question about our customer list. - """ - modules = perms.allowed_modules(session.user) - return { - # The build this Space is running. Behind the session on purpose — see `main.py::health`, - # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a - # Space secret at every deploy; absent means "somebody started this container by hand". - "version": os.environ.get("AIOS_VERSION") or "unknown", - "user": _public_user(session.user), - "scope": { - "bus": perms.allowed_bu_labels(session.user), - "team_id": perms.scope_team_id(session.user), - "agent": perms.scope_agent(session.user), - "modules": sorted(modules) if modules is not None else "all", - # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey - # pickers rather than to enforce — hidden fields are stripped from every data wire - # regardless, so a client that ignores this is narrowed anyway, never widened. - "perms": session.user.get("perms") or {}, - }, - "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", - session.tenant)}, - # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. - # - # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. - # D-160 established that a Space's environment cannot be read from outside, so "is staging - # pointed at production data?" is a question only the container can answer — and it went - # unanswered for the whole window in which two builds wrote tenant #0's real store. - # `verify_live` asserts this field after logging in, which is a check that survives a role - # swap in a way that reading a deploy log never did. - # - # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a - # dataset repo id are public names, but a health endpoint is the first URL a scanner finds - # and it may not describe the deployment. No customer data is here — only which store, and - # yes or no. - "store": _store_binding(session), - # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client - # does not paint a "Manage users" button that 403s. - "admin": perms.is_admin(session.user), - # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail - # paints its entry iff this is true, and `routes_platform_admin` refuses every request - # that is not, so a client that ignored this would gain exactly nothing. Derived from the - # ONE predicate rather than restated: a second copy of a two-condition wall is a second - # place for one of the conditions to go missing. - "platformAdmin": platform_admin.is_platform_admin(session.user), - } +"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). + +These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` +modals, over the SAME `core/users` account store — so both front-ends administer one set of +accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. + +ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit +dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record +is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. + +⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` +has no caller anywhere in the shipped app — module grants are only settable by editing the store +JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both +asserted in `verify_api.py` section A: + + * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin + clearing every checkbox to lock an account down would grant it everything. + * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to + `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose + whole point is strict isolation. + +Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in +`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the +WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both +read semantics are untouched — the seam is write-strict / read-unchanged. + +⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` +one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no +accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". +""" +import os +import re + +from fastapi import APIRouter, Body, Depends, Response + +import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag +import core.registry as registry +import core.store as store + +from deps import Session, err, perms, require_session, users +from routes_auth import _public_user + +router = APIRouter(prefix="/api/v1") + +#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under +#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no +#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. +_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") +#: ⭐⭐ WAVE 37 · T03 (owner ruling R8) — AN EMAIL ADDRESS IS ALSO A USERNAME. Owner: the username +#: IS the email, one identity. ⛔ THIS IS NOT A NEW CAPABILITY, IT IS THE DOOR CATCHING UP WITH THE +#: DATA: measured on deployed staging before the change, FIVE of tenant #0's eight accounts already +#: carry an `@` in the `username` field (`david@fischfloralsupply.com`, `farhan@`, `florencia@`, +#: `karen@`, `naomi@`) — every one of them illegal under the slug pattern above and longer than its +#: 32-character cap, so this route could not have created any of them. +#: ⚠ DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the +#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the +#: slug already trusts. A stricter grammar here would refuse addresses that exist. +_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$") +#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`. +_UNAME_MAX = 254 +#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog +#: enforces nothing; this does, and the two are allowed to differ because only one of them is +#: reachable from the internet. +_MIN_PW = 8 +_ROLES = ("user", "admin") + + +def admin_gate(session: Session = Depends(require_session)) -> Session: + """401 without a session, 403 without the admin role. Role, not a module grant — mirroring + `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + return session + + +def _require_store(): + """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" + try: + ok = store.available() + except Exception: + ok = False + if not ok: + raise err(503, "store_unavailable", + "the tenant store is unavailable — accounts cannot be read or changed") + + +def _registry(): + _require_store() + try: + reg = users.registry() or {} + except Exception: + raise err(503, "store_unavailable", "the tenant store is unavailable") + return reg + + +def _view(uname, rec, governed=None, st=None, surface_keys=None): + """What an administrator may see about an account. NEVER `salt` or `hash` — this function is + the only projection these routes use, so there is one place that can leak and it does not. + + ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both, + resolved once for the whole request; the single-record routes pass neither, because their + caller fetches the editor payload separately and `GET /perms` is the authority there. + """ + return {"username": uname, + "name": rec.get("name") or uname, + "role": rec.get("role", "user"), + "bus": rec.get("bus", "all"), + "bu_labels": perms.allowed_bu_labels(rec), + "modules": rec.get("modules", "all"), + "agent": rec.get("agent") or None, + "email": rec.get("email") or None, + "tenant": _tenant_of(rec), + "active": bool(rec.get("active", True)), + # The session-revocation handle. An admin needs it: it is the only visible evidence + # that "sign them out everywhere" actually happened. + "epoch": int(rec.get("epoch") or 0), + # S3 ask 4 — one sentence for the roster's Access column, so the list does not need + # a /perms round trip per row to fill one cell. Computed from the same record the + # editor will open, so the two cannot disagree. + "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st, + surface_keys=surface_keys), + "perms_v": int(rec.get("perms_v") or 0)} + + +def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): + """One sentence describing what this account may reach. Deliberately says LESS than the + editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters + would be wrong the moment a filter got interesting. + + ⛔⛔ W36-T22 — IT MUST COUNT WHAT THE EDITOR WOULD SHOW, NOT WHAT IS STORED, AND THE TWO STOPPED + AGREEING THE MOMENT `get_perms` LEARNED TO DEFAULT FROM `may_read`. This counted STORED ENTRIES + only, which WAS the same answer before this wave: a migrated account had an entry for every + governable database, because the editor wrote one per topic on every save and no `ut_*` entry + could exist at all. Now a `ut_*` key legitimately has NO entry and is still OPEN, so the roster + would say *"1 module"* about an account the editor shows reaching twelve. ⛔ The roster is the + screen an administrator reads FIRST, and two screens disagreeing about one account is the + defect this wave exists to remove, not a rounding difference. + + ⚠ `governed`/`st` ARE PASSED IN RATHER THAN DERIVED HERE, and that is a cost decision. This + runs once PER ROW of the roster; asking `may_read` per (account x database) without a lend + would be a whole-document copy per question — D-213's shape, one route over. The caller + resolves the tenant's list and ONE lend for the whole request. A caller that passes neither + keeps the stored-entry reading, which is right for the single-record routes: their client + fetches `GET /perms` separately and that payload is the authority. + """ + import core.perm_scope as perm_scope + + if perms.is_admin(rec): + # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if + # they applied is the one misreading of that clause that could actually hurt. + return "Everything (admin)" + if not perm_scope.is_migrated(rec): + mods = perms.allowed_modules(rec) + return "All modules (legacy rules)" if mods is None else \ + f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" + entries = (rec.get("perms") or {}) + if governed: + principal = users._public(uname, rec) if uname else rec + # ⛔ PER KIND, MIRRORING `get_perms`' OWN DEFAULTS — the first draft asked + # `nav_may_open` for EVERY key and that was a second idea of the count: for a migrated + # record with no topic entry the editor's default is `may_read` (DENY), so a blanket + # admission read would make the roster say "3 modules" about an account whose editor + # shows one box ticked. The roster's whole contract (leg 6 of the T22 section) is that + # it counts what the editor would show; a SURFACE key defaults from admission there, + # everything else from `may_read`, so this does exactly the same, key by key. + opened = [k for k in governed + if (perm_scope.nav_may_open(principal, k, st=st) + if k in (surface_keys or ()) else + perm_scope.may_read(principal, k, st=st))] + else: + opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] + restricted = sum(1 for k in opened + if isinstance(entries.get(k), dict) + and (entries[k].get("filter") or entries[k].get("hiddenFields"))) + if not opened: + return "No modules" + base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" + return f"{base}, {restricted} restricted" if restricted else base + + +# ── request validation: every rule below is fail-closed ────────────────────────────────────────── +def _clean_username(v): + """The ONE validator for a new account's key. R8 widened it to admit a full email address. + + ⛔ IT HAS EXACTLY ONE CALL SITE AND THAT IS LOAD-BEARING, not an oversight to tidy up. + `PATCH /admin/users/{username}`, the password route and both `/perms` routes take the username + as a PATH parameter and never re-validate it, which is precisely why the five email-keyed + accounts above are administrable today despite being unconstructable. Arming this validator on + those routes would look like hardening and would break five live accounts. + """ + uname = str(v or "").strip().lower() + if len(uname) > _UNAME_MAX or not (_UNAME_RE.match(uname) or _UNAME_EMAIL_RE.match(uname)): + raise err(400, "bad_username", + "a username is either 2 to 32 characters of lowercase letters, digits, dot, " + "dash or underscore, or a full email address of up to 254 characters") + return uname + + +def _clean_password(v): + pw = str(v or "") + if len(pw) < _MIN_PW: + raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + return pw + + +def _clean_role(v): + role = str(v or "").strip().lower() + if role not in _ROLES: + raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") + return role + + +def _clean_bus(v): + """'all', or a non-empty list of KNOWN business-unit ids. + + Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no + recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_bus", + "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " + "would read as unrestricted)") + out = [] + for b in v: + try: + b = int(b) + except (TypeError, ValueError): + raise err(400, "bad_bus", "a business-unit id must be a number") + if b not in users.BU_LABELS: + raise err(400, "bad_bus", + f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " + "unrecognised id would widen access to every BU") + out.append(b) + return sorted(set(out)) + + +def _clean_modules(v): + """'all', or a non-empty list of KNOWN module keys. + + `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is + refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would + silently lock the account out of the page they meant to grant. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_modules", + "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " + "reads as UNRESTRICTED, which is the opposite of locking an account down") + known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) + out, bad = [], [] + for k in v: + k = str(k or "").strip() + (out if k in known else bad).append(k) + if bad: + raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") + return sorted(set(out)) + + +def _tenant_of(rec): + """The account's company, with the pre-wave default. ONE spelling of the default, used by + every admin route — two spellings is how a tenant wall grows a gap.""" + return str((rec or {}).get("tenant") or "royal-imports").strip().lower() + + +# ── the routes ─────────────────────────────────────────────────────────────────────────────────── +@router.get("/admin/users") +def list_users(session: Session = Depends(admin_gate)): + """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global + control-plane bucket, so without this filter a Nurilab admin would read Royal's whole + roster — names, emails, agents — from one URL.""" + reg = _registry() + mine = _tenant_of(session.user) + # ⭐ W36-T22 — the tenant's governable list and ONE lend, resolved once for the whole roster so + # `_access_summary` can agree with the editor without paying a document copy per question. + import core.user_tables as _ut + try: + _rows = _perm_modules(session, surfaces=True) + governed = [m["key"] for m in _rows] + surface_keys = {m["key"] for m in _rows if m.get("surface")} + lent = _ut.lend_defs(session.runtime) + except Exception: # noqa: BLE001 + # A store that cannot list the tenant's databases must not take the roster down. The + # summary degrades to the stored-entry reading, which is what it always was. + governed, lent, surface_keys = None, None, None + return {"users": [_view(u, reg[u], governed=governed, st=lent, surface_keys=surface_keys) + for u in sorted(reg) + if _tenant_of(reg[u]) == mine]} + + +@router.post("/admin/users", status_code=201) +def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create an account. **409 if the username already exists — `PATCH` is the update path.** + + ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh + record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and + RESURRECT every cookie minted before its last password change. `core/users.py` now + preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / + update a user" calls the same function), but this route still refuses: an upsert that silently + replaces an account's password, role and BU access because someone reused a username is not an + update anybody asked for. + """ + body = body or {} + _require_store() + uname = _clean_username(body.get("username")) + pw = _clean_password(body.get("password")) + role = _clean_role(body.get("role") or "user") + bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") + modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") + name = str(body.get("name") or "").strip() or uname + agent = str(body.get("agent") or "").strip() or None + email = str(body.get("email") or "").strip() or None + + _reg_before = _registry() + if uname in _reg_before: + raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + # ⭐⭐ WAVE 37 · T03 (R8) — THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME. + # `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only + # scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was + # incidental while a username could not contain `@`; now it decides which of two accounts a + # person reaches, so it is stated in both files rather than left to be re-derived. + # ⛔ BOTH DIRECTIONS, because each one silently takes a login away from an account that has it: + # (a) a new USERNAME equal to somebody's EMAIL wins the key lookup and shadows their address; + # (b) a new EMAIL equal to somebody's USERNAME can never be reached, because their key wins. + # Refused with a named error rather than allowed and explained afterwards: the failure is a + # person signing in and landing in the wrong account, which nothing downstream can detect. + if "@" in uname: + _shadowed = next((k for k, r in _reg_before.items() + if isinstance(r, dict) + and str(r.get("email") or "").strip().lower() == uname), None) + if _shadowed: + raise err(409, "username_shadows_email", + f"{uname} is already the email address on the account {_shadowed}, which " + f"signs in with it today. This account would take that login over. Choose a " + f"different username, or clear the email address on {_shadowed} first.") + if email: + _e = email.strip().lower() + if _e != uname and _e in _reg_before: + raise err(409, "email_shadows_username", + f"{_e} is already a username on this deployment, so anyone signing in with " + f"it reaches that account and never this one. Choose a different email " + f"address for this account.") + # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — + # the tenant is stamped from the session, not taken from the body. + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email, tenant=_tenant_of(session.user)) + reg = _registry() + if uname not in reg: + # The store accepted the write and does not have it. Reporting 201 here would be the + # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. + raise err(503, "store_unavailable", "the account was not saved — try again") + return {"user": _view(uname, reg[uname])} + + +@router.patch("/admin/users/{username}") +def update_user(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Change access on an existing account. Only the keys PRESENT in the body change. + + ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every + request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very + next call — bumping the epoch would only add a forced re-login on top. `active: false` is the + exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its + session rather than keep working until the cookie expires. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if not body: + raise err(400, "empty_patch", "no fields to update") + + unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) + if unknown: + # A silently-ignored field is how a UI ends up believing it saved something it did not. + raise err(400, "unknown_fields", f"cannot update: {unknown}") + + is_self = uname == session.uname + role = _clean_role(body["role"]) if "role" in body else None + active = bool(body["active"]) if "active" in body else None + # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is + # not the thing that keeps the product reachable — it just stops an administrator removing + # their own access with a toggle and having to go find the master password. + if is_self and role is not None and role != "admin": + raise err(400, "self_demote", + "you cannot remove your own administrator role — ask another admin") + if is_self and active is False: + raise err(400, "self_deactivate", "you cannot deactivate your own account") + + kw = {} + if role is not None: + kw["role"] = role + if "name" in body: + kw["name"] = str(body["name"] or "").strip() or uname + if "bus" in body: + kw["bus"] = _clean_bus(body["bus"]) + if "modules" in body: + kw["modules"] = _clean_modules(body["modules"]) + if "agent" in body: + kw["agent"] = str(body["agent"] or "").strip() # '' clears the link + if "email" in body: + kw["email"] = str(body["email"] or "").strip() + if kw: + users.set_access(uname, **kw) + if active is not None and active != bool(reg[uname].get("active", True)): + users.set_active(uname, active) # bumps the epoch: kills live sessions + + fresh = _registry() + return {"user": _view(uname, fresh[uname])} + + +@router.post("/admin/users/{username}/password", status_code=204) +def set_password(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump + happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two + can never disagree. + + 204 with no body: there is nothing to say that the caller did not already know, and echoing the + account back would invite a client to diff a response for a password change. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + pw = _clean_password((body or {}).get("password")) + before = int(reg[uname].get("epoch") or 0) + users.set_password(uname, pw) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # The whole value of this route is the revocation. If the epoch did not move, the sessions + # were not revoked, and answering 204 would say they were. + raise err(503, "store_unavailable", + "the password change did not persist — outstanding sessions were NOT revoked") + return Response(status_code=204) + + +# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── +#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema +#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list +#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` +#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), +#: so a migrated account would be locked out of Product with no control anywhere to grant it: +#: fail-closed, but unadministrable. The editor grows its section with zero client edits +#: (S3 built it off this route's `modules` + `fields_by_module`). +#: +#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be +#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple +#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid +#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` +#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, +#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` +#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s +#: `section_tenant_derived` NC does exactly that and reds. +#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from +#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, +#: non-group_only registry row that declares a `topic`. What stays hand-written is +#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` +#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this +#: editor at all. The intersection is the honest answer to "what can this module build pickers for". +#: +#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads +#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the +#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise +#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red +#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives +#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. +_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) + + +def _governable_catalogue(): + """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. + + ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is + tenant-blind by design. Two filters in two files answering one question is how the perms route + and `routes_nav` came apart in the first place. + """ + import core.registry as registry + return [{"key": v["key"], "label": v["label"]} + for v in registry.governable_modules().values() + if v.get("key") in _FIELD_PROVIDER_KEYS] + + +_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) + + +def _perm_modules(session, surfaces=False): + """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. + + Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned + module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and + the tenant's own `ut_*` databases read through the DEFINITIONS projection. + + ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE + resolver for "which databases may this account see", it is `may_open`-filtered, and it reads + the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. + The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver + instead of assuming that is what keeps this from becoming the SECOND idea of who may see a + table (`user_tables.may_open`'s own docstring is the record of the first time that happened). + """ + import core.perms as _perms + import core.user_tables as user_tables + try: + ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, + st=session.runtime) + except Exception: + # A store that cannot list the tenant's databases must not take the whole editor down — + # the topic half is still administrable, and an empty `ut_*` half is visibly empty. + ut = () + cat = _governable_catalogue() + skeys = set() + if surfaces: + # ⭐⭐ OWNER RULING 2026-08-18 — the Assistant and Agents surfaces are + # governable ACCESS toggles in the ACCOUNT editor. `surfaces` is caller-explicit and + # defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor + # (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE + # (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle + # there would be a stored rule nothing applies, the exact class C2 deleted. + surf = _perms.governable_surface_modules() + skeys = {s["key"] for s in surf} + cat = cat + surf + rows = _perms.tenant_governable_modules(session.runtime, cat, ut_entries=ut) + # The marker survives OUTSIDE `tenant_governable_modules`, which builds bare {key,label} + # rows: the client needs it only to choose the schemaless help copy, and the server needs + # it only to pick the admission default in `get_perms`. + return [dict(r, surface=True) if r["key"] in skeys else r for r in rows] + + +#: ⛔ `_enforced_keys` IS DELETED (W36-T22 / CONTRACT C2). It asked `perms.enforced_module_keys` +#: which subset of the listed databases a wall could actually be stored against, and after R6 the +#: answer is "all of them" — a question with one possible answer is not a question. Both it and the +#: `perms` helper behind it are gone rather than made to return everything, because a predicate +#: that ignores its argument is the always-true flag C2 deletes, one indirection along. +#: What callers use instead is `{m["key"] for m in _perm_modules(session)}` — the SAME list the +#: editor renders, so "shown" and "storable" cannot drift apart again. + + +def _module_fields(key, session=None): + """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, + so an admin filters on exactly what the user sees. + + ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing + what to hide, so hiding the choices would make a field unhideable the moment it was hidden + once for somebody. The route is admin-gated; the projection a MEMBER receives is the one + `perm_scope.visible_fields` narrows. + + ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the + editor the customer schema for it would let an admin build a wall out of columns that + module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, + a filter naming a field the product rows lack would DENY every row via `permits()`. + The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader + is served fewer columns, but the ADMIN is choosing what may ever be hidden. + + ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read + `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that + was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being + a two-element literal, a `ut_*` database or a third topic would have been handed customer + columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the + stored wall would name columns that module does not have — which `permits()` resolves by + DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: + a topic without a schema arm is a missing arm, never a customer grid in disguise. + + ⭐⭐ W36-T22 / R6 — AND `ut_*` NOW HAS AN ARM, WHICH IS THE HALF THAT MAKES THE TOGGLE REAL. + Owner item 11: *"how come the database is only toggleable for Odoo customers and Odoo + products … EVERY database should be able to be toggleable by admin."* A `ut_*` database's + field contract is its OWN stored definition, so the picker offers exactly the columns that + database has. ⛔ ONE SOURCE, NOT TWO: the picker (`get_perms.fields_by_module`) and the + validator (`_clean_perms`) both call this, so a column the picker offers can never be a + column the validator rejects. Getting that wrong is worse than an error message — + `permits()` DENIES on a predicate it cannot answer, so a mismatched wall is a + deny-everything trap wearing a saved-successfully toast. + + ⚠ `session` IS REQUIRED FOR A `ut_*` KEY and unused for a topic: a stored definition is + per-tenant data and there is no tenant-blind way to read one. A caller that omits it is + refused LOUDLY rather than handed another tenant's schema — the same rule the `else` + fall-through above was deleted for. + """ + import aios_grid + key = str(key or "") + if key in {s["key"] for s in perms.governable_surface_modules()}: + # ⭐ A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest + # answer rather than the wave-26 empty-option trap: the editor's schemaless rendering + # is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any + # filter or hiddenFields validated against this empty vocabulary. An unknown key + # still RAISES below — this arm admits exactly the derived surface list. + return [] + if key.startswith("ut_"): + if session is None: + raise err(400, "ungoverned_module", + f"{key!r} is a tenant database, so its columns can only be read for a " + f"known tenant — this caller passed no session") + import core.user_tables as user_tables + defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime)) + if not defn: + raise err(400, "ungoverned_module", + f"{key!r} is not a database in this workspace") + src = defn.get("fields") or [] + else: + providers = {"customer_data": lambda: aios_grid.FIELDS, + "product_data": aios_grid.product_fields} + provider = providers.get(key) + if provider is None: + # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative + # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is + # truthy. + raise err(400, "ungoverned_module", + f"{key!r} has no permission-editor field schema. The editor governs " + f"{sorted(providers)} and this tenant's own databases") + src = provider() + known = _server_side_vocabularies() + out = [] + for f in src: + if not isinstance(f, dict) or not f.get("key"): + continue + opts = f.get("options") + if not isinstance(opts, list) or not opts: + # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See + # `_server_side_vocabularies`. + opts = known.get(f["key"]) + out.append({"key": f["key"], "label": f.get("label") or f["key"], + "type": f.get("type") or "text", + **({"options": list(opts)} if isinstance(opts, list) and opts else {}), + **({"pinned": True} if f.get("pinned") else {})}) + return out + + +def _server_side_vocabularies(): + """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, + but which the grid contract does not declare. + + ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid + resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the + field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so + every column that leans on the second branch renders `Select…` with nothing in it — the + identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix + belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied + list as authoritative *including when it is empty* ("this column has no values" is an answer), + which is correct and must not be weakened. So the caller stops handing it nothing. + + ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, + the same constant `_bucket()` mints from — a copy here would be a second vocabulary that + drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy + for the same reason `aios_grid`'s is: this route should not pull the analytics stack at + module load. + + ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than + let them look handled: + · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no + closed list to declare, and discovering it would mean a pool read on an admin route. + · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E + answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which + is the right home (the grid picks a declared list up for free) and is why this function + never needed an arm for it. Recorded because the ask, not a guess, is what settled it. + """ + try: + import modules.inventory as inventory + labels = list(inventory.COVERAGE_LABELS or ()) + except Exception: + # A missing analytics import must not take the permission editor down; the dropdown + # degrades to today's empty one rather than the page to a 500. + return {} + return {"stock_bucket": labels} if labels else {} + + +def _clean_perms(v, governed_keys=None, session=None): + """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. + + ⭐ WAVE 33 (W33-T33) — `governed_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It + used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 + flagged: an admin of a tenant provisioned for neither grid module could still store a wall + for both. It now comes from `_perm_modules(session)`, i.e. from the tenant's own catalogue. + The default is the topic catalogue itself so a test or a future caller cannot silently widen + it. + + ⭐⭐ W36-T22 / CONTRACT C2 — **THE `enforced`/`listed` SPLIT IS GONE, AND SO IS + `unenforced_module`.** This function took TWO key sets and refused anything in the gap + between them, with a message that ended *"Share the database instead, or arm `perm_scope` + over `ut_*` first."* W36-T21 armed it. There is no gap left: every database the editor lists + is a database whose wall `routes_tables` applies on every read, so a wall stored against one + is enforced by the same code that enforces `customer_data`'s. **Two sets collapse into one**, + which is the shape C2 asks for — not a second set that is always equal to the first. + + ⚠ `session` IS THREADED SO `_module_fields` CAN READ A `ut_*` SCHEMA. Without it the + validator would have no field contract for the databases this ticket just made governable, + and "no contract" resolves to "every `hiddenFields` entry is unknown" — a 400 on the admin's + own screen. + + ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately + drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION + filter is the opposite situation — an admin types "agent is Tara", a leaf is silently + dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole + book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: + the filter is re-validated with `clean_filter_tree` and the result must come back with the + SAME leaf count it went in with. + + Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` + validates the NODE LIST alone and never sees the root conjunction, so the two are checked + separately and a bare node list is refused rather than silently read as `and`). + """ + import aios_grid + + if v is None: + return None + if not isinstance(v, dict): + raise err(400, "bad_perms", "perms must be an object keyed by module") + governed = set(governed_keys) if governed_keys is not None else set(_PERM_MODULES) + unknown = sorted(set(v) - governed) + if unknown: + raise err(400, "bad_perms", + f"unknown module keys: {unknown} — this tenant's permission editor governs " + f"{sorted(governed)}") + out = {} + for key, raw in v.items(): + if not isinstance(raw, dict): + raise err(400, "bad_perms", f"{key}: each entry must be an object") + valid_keys = {f["key"] for f in _module_fields(key, session=session)} + hidden = raw.get("hiddenFields") or [] + if not isinstance(hidden, list): + raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") + bad = sorted({str(h) for h in hidden} - valid_keys) + if bad: + # A hiddenFields entry naming nothing hides nothing — and reads as a restriction + # that is not there. + raise err(400, "bad_perms", + f"{key}: hiddenFields names unknown fields {bad}") + tree = raw.get("filter") + clean_tree = None + if tree not in (None, {}, []): + if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): + raise err(400, "bad_filter", + f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " + f"the root conjunction, and an 'or' wall read as 'and' restricts " + f"nothing it was meant to") + conj = tree.get("conj", "and") + if conj not in ("and", "or"): + raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") + nodes = tree["nodes"] + cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) + if _leaf_count(cleaned) != _leaf_count(nodes): + raise err(400, "bad_filter", + f"{key}: the filter contains conditions this module cannot evaluate " + f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " + f"are per-user and cannot be a permanent rule). Refused rather than " + f"saved with the bad conditions silently removed, which would store a " + f"weaker wall than the one on screen.") + clean_tree = {"conj": conj, "nodes": cleaned} + out[key] = {"access": bool(raw.get("access", True)), + "filter": clean_tree, + "hiddenFields": sorted({str(h) for h in hidden})} + _refuse_unshapeable_bu(out) + return out + + +def _refuse_unshapeable_bu(perms_out): + """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. + + Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes + that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which + is what `perm_migrate` emits and what the condition builder produces for "is any of". An + admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a + `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through + `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries + Fisch+Royal numbers. The row list looks right. The revenue is another BU's. + + That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just + shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with + nothing anomalous about them. + + So it is refused at the WRITE, where a person is present to fix it. The discriminator is + narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like + `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and + consolidated values are the correct answer for it, so it is not caught here (`derive` also + refuses it) and must not be. + """ + import core.perm_scope as perm_scope + + for key, e in perms_out.items(): + tree = e.get("filter") + if not tree or not _mentions_dba(tree.get("nodes")): + continue + probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, + "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} + team_id, _ = perm_scope.derive_pool_scope(probe, key) + if team_id is None and _confines_to_one_bu(tree): + raise err(400, "bu_condition_shape", + f"{key}: this filter restricts the business unit in a way the data layer " + f"cannot push into the query, so the rows would be correct while the " + f"revenue figures on them stayed consolidated across both units. Express " + f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " + f"or an 'any of' over the brands — and keep any other rules alongside it.") + + +def _guard_bu_widening(rec, cleaned, confirmed): + """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. + + `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the + first save the filter IS the whole wall and the admin owns it. But the first save also + CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted + rather than the one the server returned will, on its next save, PUT a payload with no BU + condition — and a whole-record replace deletes it. Measured, not theorised: the same client + payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both + business units. + + So a save that REMOVES the brand confinement is refused unless it says it means to. Widening + BU access is a legitimate thing for an administrator to do — it is just never a thing to do + by accident, and the difference between the two is one explicit flag. + + ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU + restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the + admin edits it). The right shape is "you may widen, say so". + """ + import core.perm_scope as perm_scope + + if confirmed: + return + stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} + for key, new_entry in cleaned.items(): + if not new_entry.get("access", True): + # Revoking the module entirely is the OPPOSITE of widening, and the filter on a + # denied entry governs nothing. Guarding it would make "lock this account out" + # require a confirm_widen flag, which reads as nonsense to whoever hits it. + continue + old = stored.get(key) + if not isinstance(old, dict): + continue # nothing stored yet: the fold already handled it + old_tree, new_tree = old.get("filter"), new_entry.get("filter") + if not old_tree or not _confines_to_one_bu(old_tree): + continue # was not confined -> this save cannot widen it + if new_tree and _confines_to_one_bu(new_tree): + continue # still confined -> not a widening + raise err(400, "bu_widening", + f"{key}: this would REMOVE the business-unit restriction and give the account " + f"both units. If that is intended, resend with confirm_widen: true. If it is " + f"not, reload the account's permissions first — saving a filter loaded before " + f"the last change drops the conditions it did not know about.") + # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees + # the connection to derive_pool_scope, which is what the removed condition was feeding. + _ = perm_scope + + +def _mentions_dba(nodes): + for n in nodes or (): + if not isinstance(n, dict): + continue + if isinstance(n.get("children"), list): + if _mentions_dba(n["children"]): + return True + elif n.get("colId") == "dba": + return True + return False + + +def _confines_to_one_bu(tree): + """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? + + ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard + refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a + high-balance Royal customer satisfies it — so consolidated values are the correct answer and + refusing it would be a false positive. My first attempt evaluated synthetic rows carrying + only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact + wall into a "confinement" it is not. + + So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the + reader satisfies everything else; can they still see both brands?" If yes, the wall does not + confine and the pool is correctly consolidated. If no, the brand structure is doing the + confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. + """ + def admits(node, brand): + if isinstance(node.get("children"), list): + kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] + if not kids: + return True + return any(kids) if node.get("conj") == "or" else all(kids) + if node.get("colId") != "dba": + return True # every other condition: assume satisfied + want = str(node.get("value") or "").strip().lower() + op = node.get("op") + if op == "eq": + return brand.lower() == want + if op == "neq": + return brand.lower() != want + if op == "contains": + return want in brand.lower() + if op == "doesNotContain": + return want not in brand.lower() + if op == "isEmpty": + return False # a probe brand is never blank + if op == "isNotEmpty": + return True + return True # an op that cannot judge a brand does not confine + + nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") + root = {"conj": conj, "children": nodes} + admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} + if not admitted: + return False # admits nothing at all — a different problem + # 'Both' belongs to either unit, so it cannot by itself widen the answer. + return not ({"Fisch", "Royal"} <= admitted) + + +def _leaf_count(nodes): + n = 0 + for node in nodes or (): + if isinstance(node, dict) and isinstance(node.get("children"), list): + n += _leaf_count(node["children"]) + else: + n += 1 + return n + + +def _fold_legacy_scope(rec, cleaned): + """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. + + Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` + fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write + whose filter carries no BU condition silently promotes a Royal-only account to BOTH business + units: the wall that used to come from `bus` is gone and nothing replaced it. + + Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check + passed, because the model was doing exactly what it was told. The account simply received + the other BU's customers on the wire. + + So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as + `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a + perms block. From then on the admin sees those leaves in the editor (GET returns the folded + filter) and can change them like any other condition — which is R1's whole point, and why + this is a one-time fold rather than a permanent floor AND-ed on every write. + + Idempotent by construction: it runs only while the record is un-migrated, and the write that + calls it is the write that migrates it. + + ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE + NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` + has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent + filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into + the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for + a `bus:[5]` account the moment an admin pressed Save. + + Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against + `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a + cross-topic leaf could come through, which is why it is the only place that needs the rule. + + ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the + BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU + is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). + The pushdown falls back to the record's own `bus` for exactly the modules whose field + vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves + ship together or the second one is a leak. + """ + import core.perm_migrate as perm_migrate + import core.perm_scope as perm_scope + + if perm_scope.is_migrated(rec): + return cleaned + legacy = perm_migrate.filter_for(rec) + if not legacy: + return cleaned # nothing to preserve: the account was unrestricted + out = {} + for key, e in cleaned.items(): + legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) + tree = e.get("filter") + if not legacy_here: + # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than + # storing a wall it will read as "deny every row". + out[key] = e + continue + if not tree or not tree.get("nodes"): + folded = legacy_here + else: + # AND the two roots together. The submitted tree keeps its own conjunction by being + # nested as a GROUP — flattening an `or` tree into an `and` root would turn the + # admin's "A or B" into "A and B", which is a different and much narrower wall. + folded = {"conj": "and", + "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), + "children": list(tree["nodes"])}]} + out[key] = dict(e, filter=folded) + return out + + +def _prune_to_module(tree, valid_keys): + """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. + + Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies + every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the + reason its own docstring gives (a silently-weakened wall reads as "Saved"). + + ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than + the admin's intent and half of an `and` group is a wider one; neither is the thing that was + written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps + the surviving conditions ones somebody actually declared. + """ + if not isinstance(tree, dict): + return None + + def keep(node): + if not isinstance(node, dict): + return None + kids = node.get("children") + if isinstance(kids, list): + surviving = [keep(k) for k in kids] + if not kids or any(s is None for s in surviving): + return None + return dict(node, children=surviving) + return node if node.get("colId") in valid_keys else None + + nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] + return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None + + +@router.get("/admin/users/{username}/perms") +def get_perms(username: str, session: Session = Depends(admin_gate)): + """This account's permission block plus the field schema the editor needs to render it.""" + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + import core.perm_scope as perm_scope + + rec = reg[uname] + stored = rec.get("perms") or {} + # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. + # The PUT is a whole-record replace over this same module list, so a key declared here and + # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The + # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the + # access its legacy grant currently gives rather than a guess in either direction. + # + # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything + # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a + # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product + # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` + # has applied since wave 18) and merges the tenant's own `ut_*` databases. + modules = _perm_modules(session, surfaces=True) + # ⭐⭐ W36-T22 / C2 — EVERY listed database, not an `enforced` subset of them. See + # `perms.tenant_governable_modules` for why the flag is deleted rather than defaulted. + governed = [m["key"] for m in modules] + perms_out = {} + for k in governed: + e = stored.get(k) + # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN + # OUTAGE. `may_access` reads migrated-and-undeclared as DENY. That is right for a registry + # topic and catastrophic for a `ut_*` key, because no `ut_*` entry was STORABLE before this + # wave — so every migrated account carries none, and this route would default every one of + # them to `access: false`. + # + # ⛔ AND IT WOULD NOT HAVE STAYED A DISPLAY BUG FOR LONG. C2 also deleted the filters that + # kept `ut_*` keys OUT of the PUT body, so the editor now sends every declared module: an + # administrator opening this page, changing one unrelated filter and pressing Save would + # write an EXPLICIT `{access: false}` for all ten keychain databases — at which point + # `may_read`'s deny-only overlay fires and revokes them for real. One click, silent, + # permanent. Defaulting from the SAME evaluator the read door uses is what makes the page + # show what the user can actually open, so a save of an untouched payload is a no-op. + # ⚠ `_public(uname, rec)`, NOT the raw record. A stored record is keyed BY username in + # `users.json` and does not carry one INSIDE it, so `may_read` would hand `may_open` a + # `None` viewer and get a fail-closed False — the very outage this line exists to prevent, + # arriving through the fix for it. The same trap cost a gate double an hour earlier today. + # ⛔⛔ A SURFACE KEY DEFAULTS FROM `nav_may_open`, NOT `may_read`, AND THE + # DIFFERENCE IS A MASS REVOCATION. `may_read` on a non-`ut_` key falls through to + # `may_access`, which reads migrated-and-undeclared as DENY — correct for a topic + # (every save writes topic entries) and false for a surface, because NO surface entry + # was storable before 2026-08-18, so every migrated account carries none. The editor + # would paint Assistant unchecked for an account that opens it every day, and the next + # save of ANY unrelated change would write the explicit deny for real. `nav_may_open` + # is the evaluator the nav and the route gate actually ask, legacy fallback included, + # so the box shows what the account can reach and an untouched save stays a no-op. + _default = (perm_scope.nav_may_open(users._public(uname, rec), k) + if any(m.get("surface") and m["key"] == k for m in modules) else + perm_scope.may_read(users._public(uname, rec), k, st=session.runtime)) + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(_default), "filter": None, "hiddenFields": []} + return {"username": uname, + "perms": perms_out, + # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a + # migrated record and LEGACY on an un-migrated one: opposite meanings for identical + # JSON, so the client cannot render honestly without knowing which world it is in. + "perms_v": int(rec.get("perms_v") or 0), + # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can + # say "Everything (admin)" instead of rendering stored rules that do not apply. + "is_admin": perms.is_admin(rec), + # ⭐⭐ W36-T22 / C2 — `{key, label}` per row and NO `enforced`. It used to ride here + # so the editor could paint an apology ("Not set here") for a database whose wall was + # never applied; W36-T21 armed every one of them, so the flag would now be constantly + # true and the apology would be a lie with a green gate behind it. The LABEL comes + # from the registry / the table's own definition. + "modules": modules, + # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav + # schema envelope; the editor needs the field list and nothing else, and one shape + # beats two readings of a sentence. + # ⭐ EVERY governed database now carries its fields, `ut_*` included — that is owner + # item 11's *"EVERY database should be able to be toggleable by admin"*, and it is + # the SAME resolver `_clean_perms` validates against, so the picker cannot offer a + # column the validator will reject. + "fields_by_module": {k: _module_fields(k, session=session) for k in governed}} + + +@router.put("/admin/users/{username}/perms") +def put_perms(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this account's permission block wholesale. + + WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot + express a deletion without a second vocabulary for it. + + ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: + `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the + target's LIVE session on its very next call. Bumping would only add a forced re-login on top + of a change that has already taken effect. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + + # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses + # perm_scope entirely, so this cannot brick them today — but it stops an administrator + # writing a wall for themselves that would bite the moment their role changed. + if uname == session.uname: + raise err(400, "self_perms", + "you cannot set permissions on your own account — ask another admin") + + _mods = _perm_modules(session, surfaces=True) + # ⭐⭐ W36-T22 / C2 — ONE key set. It was two ("listed" and "enforced") with a refusal in the + # gap; W36-T21 closed the gap, so every database the editor shows is one whose wall the table + # routes apply. `session` rides so a `ut_*` schema can be read for THIS tenant. + cleaned = _clean_perms(body.get("perms"), + governed_keys={m["key"] for m in _mods}, session=session) or {} + cleaned = _fold_legacy_scope(reg[uname], cleaned) + _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) + users.set_access(uname, perms=cleaned) + fresh = _registry() + rec = fresh.get(uname) or {} + if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: + # The store took the write and did not record it. A 200 here would tell an administrator + # the wall is up when it is not — the one direction this must never fail in. + raise err(503, "store_unavailable", + "the permissions were not saved — the account is UNCHANGED") + return {"username": uname, "perms": rec.get("perms") or {}, + "perms_v": int(rec.get("perms_v") or 0)} + + +def _store_binding(session): + """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. + + ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 + rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset + repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path + that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only + tenant #0. Reporting the process default here would show "staging store, all fine" to exactly + the tenants that were not fine. + + ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it + could not describe the store is worse than one that says it does not know. + """ + try: + import core.data_binding as data_binding # noqa: PLC0415 + import core.store as store # noqa: PLC0415 + bound = getattr(session.runtime, "store_handle", None) + repo = getattr(bound, "repo", None) or store.REPO + return data_binding.describe(repo) + except Exception as e: # noqa: BLE001 + return {"deployment": "unknown", "production": False, "repo": "", + "writable": False, "refusal": f"could not resolve the store binding: {e}", + "localOverride": False, "allowlist": [], "observed": {}} + + +@router.get("/settings") +def settings(session: Session = Depends(require_session)): + """The session's OWN settings — any authenticated user, not admin-only. + + `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a + client may know about itself (never a hash, never the epoch). `scope` restates it as the two + things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others + would answer a question about our customer list. + """ + modules = perms.allowed_modules(session.user) + return { + # The build this Space is running. Behind the session on purpose — see `main.py::health`, + # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a + # Space secret at every deploy; absent means "somebody started this container by hand". + "version": os.environ.get("AIOS_VERSION") or "unknown", + "user": _public_user(session.user), + "scope": { + "bus": perms.allowed_bu_labels(session.user), + "team_id": perms.scope_team_id(session.user), + "agent": perms.scope_agent(session.user), + "modules": sorted(modules) if modules is not None else "all", + # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey + # pickers rather than to enforce — hidden fields are stripped from every data wire + # regardless, so a client that ignores this is narrowed anyway, never widened. + "perms": session.user.get("perms") or {}, + }, + "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", + session.tenant)}, + # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. + # + # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. + # D-160 established that a Space's environment cannot be read from outside, so "is staging + # pointed at production data?" is a question only the container can answer — and it went + # unanswered for the whole window in which two builds wrote tenant #0's real store. + # `verify_live` asserts this field after logging in, which is a check that survives a role + # swap in a way that reading a deploy log never did. + # + # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a + # dataset repo id are public names, but a health endpoint is the first URL a scanner finds + # and it may not describe the deployment. No customer data is here — only which store, and + # yes or no. + "store": _store_binding(session), + # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client + # does not paint a "Manage users" button that 403s. + "admin": perms.is_admin(session.user), + # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail + # paints its entry iff this is true, and `routes_platform_admin` refuses every request + # that is not, so a client that ignored this would gain exactly nothing. Derived from the + # ONE predicate rather than restated: a second copy of a two-condition wall is a second + # place for one of the conditions to go missing. + "platformAdmin": platform_admin.is_platform_admin(session.user), + } diff --git a/api/routes_geo.py b/api/routes_geo.py new file mode 100644 index 0000000000000000000000000000000000000000..25575695d4a054d4e567113792da75a94b75a56a --- /dev/null +++ b/api/routes_geo.py @@ -0,0 +1,369 @@ +"""routes_geo.py — the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37). + +⭐ WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor. + +R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is +retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding. +The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite, +so every provider is a (base URL + attribution) pair read from the environment and served from ONE +door. The renderer receives a URL template and a credit line; it knows nothing about who is behind +them. That is the seam. + + tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png + geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search + routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/ + +⛔ R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT. +R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners +and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly +forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a +BLOCKED tenant. Three mechanisms enforce it, none of them advisory: + + 1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so + there is no code path from "a database exists" to "its rows were geocoded". Somebody had to + choose the records. + 2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client + drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment + cannot satisfy. + 3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that + ignores everything above cannot exceed the published rate. + +⭐ AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory +for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a +re-run of the same records free rather than merely legal. + +⚠ NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which +is what every OSM client does and what the tile policy expects. Proxying them through this app +would put a free tier in the path of every pan and would breach the same policy it looks like it is +respecting. This door serves the tile URL, never the tile. +""" +import os +import re +import threading +import time +from collections import OrderedDict + +import requests +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +# --------------------------------------------------------------------- the seam +# +# Every value below is an environment override with a keyless default. Pointing the map at a paid +# vendor is three variables on the Space and no code change, which is what makes R14 a deferral. + +_TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png" +_TILE_CREDIT_DEFAULT = "© OpenStreetMap contributors" +_TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright" +_GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search" +_ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/" + +#: Both policies require a real, identifying User-Agent. A generic one is how a shared service +#: decides an application is a scraper, so this carries the product name and a contact URL. +_UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)" + +#: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than +#: sitting exactly on the published edge. +_MIN_INTERVAL_MS_DEFAULT = 1100 + +#: The most addresses one call may carry. Small on purpose: the client loops and shows progress, +#: and no single request can sit on the connection for a minute waiting out the throttle. +MAX_BATCH = 25 + +#: Tile zoom the provider actually serves. OSM stops at 19. +_TILE_MAX_Z_DEFAULT = 19 + +_CACHE_MAX = 4096 + + +def _env(name, default): + return (os.environ.get(name) or "").strip() or default + + +def _env_int(name, default): + try: + n = int((os.environ.get(name) or "").strip()) + return n if n > 0 else default + except (TypeError, ValueError): + return default + + +def provider_config(): + """The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it.""" + return { + "tiles": { + "url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT), + "attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), + "attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT), + "maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT), + }, + "geocode": { + "available": True, + "maxBatch": MAX_BATCH, + "minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT), + # ⭐⭐ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT. + # T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is + # `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of + # LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the + # limit would promise a minute and take three, and a progress estimate that runs out + # before the work does reads as a hang rather than as a wait. + "secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")), + "attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), + }, + "route": { + "available": True, + "attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"), + }, + } + + +# ------------------------------------------------------------------ the throttle +# +# ⚠ ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the +# rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each +# stay honest and together double the published rate. + +_gate = threading.Lock() +_last_call = [0.0] + + +def _throttle(min_interval_ms): + """Block until the minimum interval since the last outbound call has elapsed. + + Returns the seconds actually waited, so a caller can report the wait rather than hide it.""" + wait = 0.0 + with _gate: + gap = min_interval_ms / 1000.0 + now = time.monotonic() + due = _last_call[0] + gap + if now < due: + wait = due - now + time.sleep(wait) + _last_call[0] = time.monotonic() + return wait + + +# --------------------------------------------------------------------- the cache + +_cache = OrderedDict() +_cache_lock = threading.Lock() + + +def _cache_key(q, country): + return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower()) + + +def _cache_get(key): + with _cache_lock: + if key not in _cache: + return None + _cache.move_to_end(key) + return _cache[key] + + +def _cache_put(key, value): + with _cache_lock: + _cache[key] = value + _cache.move_to_end(key) + while len(_cache) > _CACHE_MAX: + _cache.popitem(last=False) + + +def geocode_one(address, country=None, timeout=12): + """One address to {lat, lon, label} or None. + + ⛔ THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather + than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a + second caller. A cache hit costs no request and no wait.""" + q = re.sub(r"\s+", " ", str(address or "")).strip() + if not q: + return None + key = _cache_key(q, country) + hit = _cache_get(key) + if hit is not None: + return dict(hit) if hit else None + + cfg = provider_config()["geocode"] + _throttle(cfg["minIntervalMs"]) + params = {"q": q, "format": "jsonv2", "limit": 1} + if country: + params["countrycodes"] = str(country).strip().lower() + try: + r = requests.get( + _env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT), + params=params, + headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT), + "Accept": "application/json"}, + timeout=timeout, + ) + r.raise_for_status() + rows = r.json() + except Exception: + # ⚠ A transport failure is NOT cached. Caching it would turn one bad minute into a + # permanently empty column that no re-run could ever repair. + return None + if not isinstance(rows, list) or not rows: + # A genuine "no such place" IS cached, as a negative: asking again gets the same answer and + # spends another second of a shared service's budget to hear it. + _cache_put(key, {}) + return None + top = rows[0] or {} + try: + lat = float(top.get("lat")) + lon = float(top.get("lon")) + except (TypeError, ValueError): + _cache_put(key, {}) + return None + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + _cache_put(key, {}) + return None + out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)} + _cache_put(key, out) + return dict(out) + + +# ---------------------------------------------------------------------- the doors + + +@router.get("/geo/providers") +def geo_providers(session: Session = Depends(require_session)): + """What the map should draw with, and who to credit for it. + + Session gated like every other door here. The values are not secret, but an unauthenticated + endpoint on this app is a door somebody eventually hangs something else on.""" + return provider_config() + + +@router.post("/geo/geocode") +def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)): + """Turn a bounded list of addresses into coordinates. + + ⛔ Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of + request that means "geocode everything", which is R3 expressed as an API rather than as a + warning.""" + items = body.get("addresses") + # ⭐ THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus + # the geocode field's config instead of finished address strings, and the server builds the + # query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function + # on the client, and an address composer that exists twice will disagree the first time either + # copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap + # B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot. + if items is None: + records = body.get("records") + cfg = body.get("config") or {} + if isinstance(records, list) and records: + import ai_enrich + items = [] + for rec in records: + if not isinstance(rec, dict): + continue + q = ai_enrich.geocode_query(rec.get("row") or {}, cfg) + # ⛔ A record with no usable address is REPORTED, not dropped: it comes back + # `found: false` with an empty address, so the caller can name it on screen instead + # of quietly returning fewer answers than it asked questions. + items.append({"key": rec.get("key"), "address": q}) + if not body.get("country") and cfg.get("country"): + body = {**body, "country": cfg.get("country")} + if not isinstance(items, list) or not items: + raise err(400, "no_addresses", "Send at least one address to look up.") + if len(items) > MAX_BATCH: + raise err( + 400, + "batch_too_large", + f"Look up at most {MAX_BATCH} addresses per request. " + f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.", + ) + country = body.get("country") + out = [] + started = time.monotonic() + for raw in items: + if isinstance(raw, dict): + key, addr = raw.get("key"), raw.get("address") + else: + key, addr = None, raw + # ⛔ AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a + # budget shared with everyone else using it, and asking it to place "" would spend a second + # of that budget to be told nothing. `reason` is what lets the surface say "no address on + # file" rather than "not found", which are different facts and want different actions. + if not str(addr or "").strip(): + out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None, + "label": None, "reason": "no_address"}) + continue + hit = geocode_one(addr, country=country) + out.append({"key": key, "address": str(addr or ""), "found": bool(hit), + "reason": None if hit else "not_found", + "lat": hit["lat"] if hit else None, + "lon": hit["lon"] if hit else None, + "label": hit["label"] if hit else None}) + elapsed = time.monotonic() - started + return {"results": out, + "found": sum(1 for r in out if r["found"]), + "asked": len(out), + "secondsElapsed": round(elapsed, 2), + "attribution": provider_config()["geocode"]["attribution"]} + + +@router.post("/geo/route") +def geo_route(body: dict = Body(...), session: Session = Depends(require_session)): + """Road distance, duration and the drawn line, for stops ALREADY put in order. + + ⭐ THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client, + for free, and keeps working when this service does not answer. This door adds the half that + arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start + picker and the round-trip toggle keep working with no network at all.""" + stops = body.get("stops") + if not isinstance(stops, list) or len(stops) < 2: + raise err(400, "too_few_stops", "A route needs at least two stops with a location.") + if len(stops) > 25: + raise err(400, "too_many_stops", + "Route at most 25 stops at once. Narrow the selection and try again.") + pairs = [] + for s in stops: + try: + lat, lon = float(s["lat"]), float(s["lon"]) + except (TypeError, ValueError, KeyError, IndexError): + raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.") + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.") + pairs.append(f"{lon:.6f},{lat:.6f}") + + base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT) + _throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT)) + try: + r = requests.get( + base.rstrip("/") + "/" + ";".join(pairs), + params={"overview": "simplified", "geometries": "geojson", "steps": "false"}, + headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)}, + timeout=20, + ) + r.raise_for_status() + data = r.json() + except Exception: + raise err(502, "route_service_unavailable", + "The routing service did not answer. The stop order and the straight line " + "distance are still on the map.") + routes = (data or {}).get("routes") or [] + if not routes: + raise err( + 502, + "no_route", + "No road route connects these stops. They may be on different land masses, or one of " + "them may be far from any road.", + ) + top = routes[0] + geom = ((top.get("geometry") or {}).get("coordinates")) or [] + line = [] + for c in geom: + try: + line.append([float(c[0]), float(c[1])]) + except (TypeError, ValueError, IndexError): + continue + return { + "km": round(float(top.get("distance") or 0.0) / 1000.0, 1), + "minutes": int(round(float(top.get("duration") or 0.0) / 60.0)), + "line": line, + "attribution": provider_config()["route"]["attribution"], + } diff --git a/api/routes_nav.py b/api/routes_nav.py index 388473b4c8090629c965dbe8ef85b545a9dc46c7..ced7accb1d177d570b7f3b295f3303590d9017a7 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -1,897 +1,926 @@ -"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. - -SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who -may see what — a nav that hides a link the API would still serve is a UI courtesy, not a -permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), -so the nav and the 403 can never disagree about a grant. - -The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no -`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the -`prefs.json` Library preference is deliberately NOT applied — is documented on -`core.perms.nav_pages`, which is where it belongs: one place, both callers. -""" -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, perms, require_session - -router = APIRouter(prefix="/api/v1") - -#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder -#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a -#: surface (the server-filtered nav still decides what exists). -_NAV_PREFS_KEY = "nav_prefs" -_MAX_NAV_FOLDERS = 16 - -#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. -#: -#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a -#: separate bucket rather than another field in that one. Folders and placement are one -#: person's arrangement of their own rail — per-user by definition. What a database is CALLED -#: and what it looks like are facts about the database: a workspace where two people call the -#: same table different things has no shared vocabulary left to discuss it in. Same store, two -#: buckets, because they answer to two different owners. -#: -#: Shape: {"": {"icon": {"shape": , "tone": }, -#: "name": ""}} -_NAV_META_KEY = "nav_meta" - -#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these -#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, -#: so it is the one place the list is written twice. -#: -#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by -#: shape and maps the result, so a shape this whitelist let through and the renderer does not -#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the -#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of -#: that. If the two lists ever drift the symptom is an icon that silently reverts to the -#: default, which is the loudest SAFE failure available here. -_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", - "chart", "map", "users", "clock", "heart", "bolt"}) -_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) -_MAX_NAV_NAME = 60 - -#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. -#: -#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is -#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature -#: rather than a convenience one. -#: -#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. -#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An -#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one -#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls -#: off the list while forty slots hold repeat visits to the first. Keying by page makes -#: "recent" mean what the word means, and makes the cap bound the number of DATABASES -#: remembered rather than the number of clicks. -_NAV_RECENTS_KEY = "nav_recents" -_MAX_RECENTS = 50 - -#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a -#: precedent, not a preference. `user_tables.create` writes -#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses -#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 -#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — -#: with nothing to go red, because both ends are internally consistent. An integer instant has -#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect -#: shipped; this is that lesson applied before it.) -def _now() -> int: - return int(time.time()) - - -def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): - """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. - - `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown - or invisible key is dropped (it can return when the grant does — placement is cosmetic, - so pruning is loss-free). Unknown folder refs drop the placement, not the folder. - - `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the - `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than - pruned: keeping a placement whose database may since have been deleted is cosmetically - harmless, while pruning a live one is the silent data loss this function just stopped - causing. Never widened to non-`ut_` keys — those come from the compiled registry, which - cannot fail to enumerate. - """ - raw = raw if isinstance(raw, dict) else {} - folders, seen = [], set() - for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:40] - name = " ".join(str(f.get("name") or "").split())[:40] - if not fid or fid in seen or not name: - continue - seen.add(fid) - folders.append({"id": fid, "name": name}) - ids = {f["id"] for f in folders} - placement = {} - src = raw.get("placement") - if isinstance(src, dict): - for k, v in src.items(): - k, v = str(k)[:60], str(v)[:40] - if v not in ids: - continue # the folder is gone; the placement goes with it - if page_keys is None or k in page_keys: - placement[k] = v - elif keep_unknown_ut and k.startswith("ut_"): - placement[k] = v - return {"folders": folders, "placement": placement} - - -def _placeable_top_keys(session): - """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. - - ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by - construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and - `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's - user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. - So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement - naming one — on READ and on WRITE. - - The symptom was "a database I drag into a folder falls out of it later", never "it does not - work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy - (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. - The next reload served the pruned copy and the database was back at root. Built-in modules - (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly - why it read as intermittent rather than as a missing feature. - - ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents - block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the - ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 - fix did not reach. - - `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set - as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement - because the store blinked would be the original defect wearing a different cause. - """ - pages = perms.nav_pages(session.user) or [] - keys = {p["key"] for p in pages if not p.get("parent")} - try: - import core.user_tables as user_tables - # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never - # name a database this session cannot see, and the nav and this door agree by sharing - # one predicate rather than by two lists being kept in step. - # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is - # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires - # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured - # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the - # owner reports as "the Automation row arrives ten seconds late". - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - keys.add(str(e.get("key") or "")) - except Exception: - return keys, False - return keys, True - - -def _clean_icon(raw): - """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" - if not isinstance(raw, dict): - return None - shape, tone = raw.get("shape"), raw.get("tone") - if shape not in _ICON_SHAPES or tone not in _ICON_TONES: - return None - return {"shape": shape, "tone": tone} - - -def _read_nav_meta(runtime): - """The tenant's `nav_meta`, validated on the way OUT as well as in. - - Re-validating a read looks redundant and is not: the bucket outlives this code, an older - build may have written a shape this one no longer accepts, and the whitelist is mirrored - from a vocabulary that lives in another language. A row whose icon does not survive - validation renders the default mark — never a crash, and never a half-drawn glyph. - """ - try: - stored = runtime.get(_NAV_META_KEY) or {} - except Exception: - return {} # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return {} - out = {} - for key, meta in stored.items(): - if not isinstance(meta, dict): - continue - entry = {} - icon = _clean_icon(meta.get("icon")) - if icon: - entry["icon"] = icon - # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write - # route refuses a name for a built-in key, but a record written by an older build (or - # by hand) could still carry one — and honouring it would let the store rename - # `customer_data` on screen while `core/registry.py` and every other reader went on - # calling it Customer. A name that could not be written today is not served today. - name = meta.get("name") - if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): - entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] - if entry: - out[str(key)] = entry - return out - - -def _read_recents(runtime, uname, allowed): - """This user's recents, newest first, PRUNED to what they may currently see. - - ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: - - · the WRITE happens on every route open — it is the one hot path this file has — so it - must not build the whole nav to validate one key; - · a key whose grant was REVOKED must stop being offered without anyone running a - migration, and must come back if the grant does. That is `_clean_nav_prefs`' own - prune-never-invent posture, applied to a second bucket for the same reason. - - A key that is not in `allowed` therefore reaches the store and never reaches a screen — - which also means a stuffed key cannot be used to discover what exists: it comes back only - if the session could already see it. - """ - try: - stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} - except Exception: - return [] # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return [] - out = [] - for key, at in stored.items(): - key = str(key) - if allowed is not None and key not in allowed: - continue - try: - at = int(at) - except (TypeError, ValueError): - continue # a stamp this build cannot read is not a stamp - if at <= 0: - continue - out.append({"key": key, "at": at}) - out.sort(key=lambda r: r["at"], reverse=True) - return out[:_MAX_RECENTS] - - -# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── -# -# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the -# block that computed them (`_important_counts`) went with them. It read one store bucket PER -# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database -# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on -# **14** pages. It is **0 and 0** now. -# -# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the -# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold -# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. -# A budget that can be exhausted is a number that can be silently short, which is why it also had to -# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to -# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge -# instead of a late rail. Closes D-288 and D-289. -# -# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the -# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store -# read was — and duplicating them into the new route would have put two answers behind one badge. - -#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the -#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> -#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is -#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second -#: copy here would be free to drift from it ([[one-question-two-normalizers]]). -_WS_SUFFIX = "_table_workspace" - - -def _visible_views(doc, uname, is_admin, granted_ids): - """Every view on ONE database that THIS caller can see, from an already-read workspace doc. - - ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get - wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` - — one home per view, never both — so "how many views are marked important here" has a - DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and - `admin` has none, on the same database. A tenant-wide count would put a number in the owner's - rail that no view sidebar they can open would ever add up to. - - ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for - it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole - bucket per database (the N whole reads this block exists to avoid), and the only other option is - a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told - exists; a private import cannot. - """ - import core.table_store as table_store - out = {} - for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): - if isinstance(view, dict): - out[str(vid)] = view - for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): - if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): - out.setdefault(str(vid), view) - # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the - # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. - if granted_ids: - for stratum, blob in doc.items(): - if stratum == uname or not isinstance(blob, dict): - continue - for vid, view in (blob.get("views") or {}).items(): - if str(vid) in granted_ids and isinstance(view, dict): - out.setdefault(str(vid), view) - return out - - -def _view_record_count(cfg, cohorts): - """How many RECORDS this view resolves to, or None when that cannot be answered for free. - - ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact - because the view CARRIES its row set; the third — an ordinary filtered view — can only be - counted by running its filters over the records, and the records are the one thing this route - must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` - here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and - the database's `partial` flag says so, which is the whole of R6's second sentence applied to a - badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a - number that is short by an unknown amount. - - ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` - counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); - this end is the exact inverse — it has no rows at all and no window either. The two are honest - about different halves, which is why `partial` had to be on the wire rather than derived. - """ - if not isinstance(cfg, dict): - return None - # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` - # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. - lock = str(cfg.get("cohortLock") or "").strip() - if lock: - n = cohorts.get(lock) - return int(n) if isinstance(n, int) else None - # A curated row set carries its own count. - pids = cfg.get("memberPids") - if isinstance(pids, list) and pids: - return len(pids) - return None - - -def _visible_database_keys(session): - """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. - - ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: - - · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT - CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. - A star scan cannot borrow that: it would open the view bucket of a database this workspace's - catalogue does not include. `nav()` applies the catalogue filter; that helper never has. - · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so - it cannot be reached from another route without rebuilding the payload. - - So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same - order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the - tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes - the answer honest for a caller about to read a view bucket: a module surface with no workspace - (`sales`, `ar`) is not a database and has no views to star. - - ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` - excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: - a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even - though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious - reading — this ticket's first draft assumed the clause was excluding children and wrote a - docstring around a defect that does not exist. - - ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as - authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. - """ - import core.view_templates as view_templates - keys, enumerated = set(), True - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} - for p in (perms.nav_pages(session.user) or []): - key = str(p.get("key") or "") - if allowed is not None and key not in allowed \ - and str(p.get("parent") or "") not in allowed: - continue - if view_templates.workspace_key(key): - keys.add(key) - try: - import core.user_tables as user_tables - # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection - # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - k = str(e.get("key") or "") - if k: - keys.add(k) - except Exception: - enumerated = False - return keys, enumerated - - -def _granted_view_ids(session): - """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. - - ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above - takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with - the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to - answer "which views has this person starred", so the derivation moves here beside the reader - it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). - - ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming - me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to - agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test - would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so - it costs a dict lookup per id. - ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. - """ - import core.shares as shares - try: - return {str(v) for v in - ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) - .get("view") or []) - if shares.role_for("view", str(v), session.uname, - st=session.runtime) in ("view", "edit")} - except Exception: - return set() - - -@router.get("/nav") -def nav(session: Session = Depends(require_session)): - """`{pages: [{key,label,source?,chrome}], landing}`. - - Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you - are not allowed". A session that may open nothing at all is a misconfigured account, and it - gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is - how a permission bug hides in plain sight (X2's never-an-empty-200 rule). - """ - pages = list(perms.nav_pages(session.user) or []) - # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. - # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything - # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is - # its own databases, which is what "different set of databases at later waves" means. - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. - # - # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all - # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs - # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a - # row it removes and a row a store failure dropped are the SAME absence on the wire, and the - # client renders `null` for both — pixel-identical to still-loading. Naming the removals is - # what lets the shell tell "not part of this workspace" from "we could not read it". - _omitted = [] - if tmods != "all": - allowed = {str(k) for k in (tmods or [])} - _omitted = sorted({str(p.get("key")) for p in pages - if p.get("key") not in allowed - and (p.get("parent") or "") not in allowed}) - pages = [p for p in pages - if p.get("key") in allowed or (p.get("parent") or "") in allowed] - # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — - # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so - # the nav cannot offer a row the table routes would refuse. - # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. - # - # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one - # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below - # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms - # in-process** — the second figure is the one that matters, because with the network gone and - # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every - # other collapses to 20–161 ms. That is per-call CPU, and this is it. - # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ - # manage loop below consumes this same dict. - _ut_defs, _ut_locked_mode, _degraded = {}, "", [] - try: - import core.user_tables as user_tables - # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below - # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and - # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the - # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of - # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms - # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds - # apart because of CSS or ordering, they arrived when their route finished copying. - # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that - # needs a row, it raises with the reason instead of painting an empty grid. Use - # `all_tables` for that, and know you are buying the whole copy back. - _ut_defs = user_tables.all_defs(st=session.runtime) or {} - # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its - # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked - # the moment this import failed. A sentinel that can equal real data is not a sentinel. - _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) - _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) - # ⛔ AND THE EDITOR'S DENY REACHES THIS HALF TOO. `nav_entries` is `may_open`-filtered - # — creator, admin or a share — which knows nothing about the access toggle in - # Manage user. An administrator unticking a shared database left it on the rail. - # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. - import core.perm_scope as _perm_scope - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=_lent): - if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): - continue - pages.append({"key": e["key"], "label": e["label"], - "source": e.get("source") or "Blank", "chrome": "main"}) - except Exception: - # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. - # - # A store blip must not take the whole nav down with it, and that half stands. What - # changed is that it used to answer **200 OK with every database missing** and say - # nothing — so a client cannot tell "this tenant has no databases" from "we could not - # read them", and the rail renders the same complete-looking thing either way. That is - # the owner's *"Connectors and Automation module still disappears"* class of report: - # the payload is a claim about what exists, and a claim it could not verify has to be - # marked as such. `degraded` is that mark; the shell renders it in the affected slot. - _degraded.append("databases") - pass - if not pages: - if tmods != "all": - # A provisioned tenant with no modules and no databases YET is a legitimate empty - # state, not a misconfigured account — the client renders "create your first - # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is - # empty rather than leaving 200-[] ambiguous. - return {"pages": [], "landing": None, "empty": "no_databases"} - raise err(403, "no_surfaces", - "your account has no dashboards assigned. Ask an administrator.") - # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry - # rows, after the tenant module filter, after the user tables. Merged HERE rather than - # applied by the client for one reason: `label` is what every reader of this payload shows, - # including the Settings modal's `moduleLabels` map, so a client-side merge would put the - # override in one door and the registry label in the other. - # - # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A - # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If - # either ever starts handing back cached or module-level dicts, this loop would write one - # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant - # leak with no symptom until two tenants rename the same registry key. Copy the rows before - # merging on the day that invariant changes. - meta = _read_nav_meta(session.runtime) - # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. - # - # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS - # ORDER so a cold container would reach the databases somebody actually works in first. Every - # one of those was a correct answer to the wrong question: the block was `N` per-database store - # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block - # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per - # `/nav`** before this ticket, **0 and 0** after. - # - # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer - # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. - # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave - # 23), and it was only computed this early so the deleted budget could spend itself in its order. - _page_keys = {str(p.get("key", "")) for p in pages} - recents = _read_recents(session.runtime, session.uname, _page_keys) - # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. - # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. - # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, - # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. - # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an - # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. - for p in pages: - # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because - # the server is the only end that knows — the client cannot see who created a user - # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control - # only where the write would actually land, and the route re-checks it regardless. - # - # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the - # de5037f share-grant admission widened `may_open`, so a row's presence now includes - # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are - # therefore answered from the DEFINITION: creator-or-admin strictly, matching the - # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no - # controls — the honest shape (the old code offered rename to users the route 403'd). - key = str(p.get("key", "")) - if key.startswith("ut_"): - _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") - _mine = bool(session.admin or (_creator and _creator == session.uname)) - p["manage"] = _mine - p["canDelete"] = _mine - # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. - # - # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md - # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still - # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the - # live example; their rows arrive from the engine, so a "+" row there could only - # refuse, which R8 calls a fake affordance. - # - # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is - # inlined only because `_ut_defs` is already in hand — calling the predicate per row - # would be one store read per database on every nav request — and the VALUE comes - # from the module's own constant rather than a copied string, so the two cannot drift - # to different answers. If that predicate ever grows a second condition, this must - # become a call. - # - # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is - # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while - # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs - # a missing hint, never a write that should not have landed. - if (_ut_locked_mode - and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): - p["locked"] = True - elif session.admin: - p["manage"] = True - # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database - # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer - # never had to test for the key. R4 retires the count from the rail and the flyout entirely - # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. - # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an - # absent key reads as `undefined` there, which is the same thing the optional field already - # meant for a non-database row. Flagged to A rather than assumed harmless. - entry = meta.get(p.get("key")) if meta else None - if not entry: - continue - if entry.get("icon"): - p["icon"] = entry["icon"] - if entry.get("name"): - p["label"] = entry["name"] - landing = perms.landing_page(session.user) - if landing and not any(p.get("key") == landing for p in pages): - landing = pages[0].get("key") - # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks - # for. A second round trip for a list this short, computed from a bucket this route is - # already holding the store open for, would be a request per page load for no gain. - # - # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases - # included. Pruning against `perms.nav_pages` alone would silently drop every user database - # from Home's recents: the exact surface R7 is about, invisible, with every gate green. - # - # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — - # had the identical bug and nobody connected the two for four waves. It is fixed at the - # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so - # both doors finally agree on what a placeable key is. This block keeps using the assembled - # list because it already holds it: re-enumerating here would be a second store read for an - # answer sitting in a local variable. - # - # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, - # because the important-count block spends its budget in RECENTS ORDER (see there). It is still - # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here - # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. - # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. - # `omitted` — this workspace's catalogue does not include these modules. Deliberate. - # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says - # so in the affected slot instead of rendering a confident nothing. - # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as - # `[]` rather than being omitted when empty, which is the same rule `limits` follows. - return {"pages": pages, "landing": landing, "recents": recents, - "omitted": _omitted, "degraded": _degraded} - - -@router.post("/nav/opened") -def nav_opened(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. - - The client calls this on every route commit, so this is the only write in this file on a - hot path, and three things follow from that: - - · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per - page open against an HF-Dataset-backed store would put a network round trip inside every - navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; - this is. - · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string - would make the stamp cost more than the navigation that triggered it — and it would buy - nothing, because the READ prunes to what the session may currently see. An unknown or - revoked key is stored and never served back. - · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client - grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to - go, which is the same rule the read applies. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip()[:60] - if not key: - raise err(400, "bad_request", "no page was named") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was recorded.") - stamp, uname = _now(), session.uname - - def _up(data): - data = data if isinstance(data, dict) else {} - mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} - mine[key] = stamp - if len(mine) > _MAX_RECENTS: - # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape - # sorts as 0 and is the first thing evicted, which is the right answer for a value - # this route can no longer read. - def _at(item): - try: - return int(item[1]) - except (TypeError, ValueError): - return 0 - mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) - data[uname] = mine - return data - - try: - session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was recorded.") - return {"key": key, "at": stamp} - - -@router.post("/nav/meta") -def save_nav_meta(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. - - A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the - asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so - replacing the document whole is what makes a deleted folder stay deleted. This bucket is - shared by every admin in the tenant: a wholesale write here means whoever saves last - silently erases what the other one named while their tab was open. - - THREE WALLS, all fail-closed: - · the SESSION must be able to open the key at all (the same predicate the nav uses, so - the rail and this route cannot disagree about what exists); - · the WRITE is admin-only — renaming a database is a change every user in the tenant - sees, which is the definition of an administrative act here; - · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry - literals: honouring an override would leave this payload and `core/registry.py` - calling the same module two different things, and the wave-16 lesson is that the - client must not be the place that decides what a payload meant. - - `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an - icon change two independent writes rather than a race between them. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip() - if not key: - raise err(400, "bad_request", "no database was named") - # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── - # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. - # - # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what - # `user_tables.may_open` already means. A database you made is yours to name; requiring - # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` - # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, - # because a user table is not a module. Same split `/nav/schema/{key}` makes. - # · everything else — admin only, and icon only. There is no owner of `customer_data` to - # defer to, and its label is a compiled registry literal (refused below regardless). - if key.startswith("ut_"): - import core.user_tables as user_tables - if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( - key, session.uname, session.admin, st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - else: - if not session.admin: - raise err(403, "forbidden", - "only an administrator can change a built-in database's icon") - session.require(key) - - patch = {} - if "icon" in body: - icon = _clean_icon(body.get("icon")) - if body.get("icon") is not None and icon is None: - # Loud, not silent. A shape this build does not know is a CLIENT that has drifted - # from this whitelist, and answering 200 to a write that stored nothing is how - # that drift stays invisible until a user reports "my icon keeps resetting". - raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") - patch["icon"] = icon - if "name" in body: - if not key.startswith("ut_"): - raise err(400, "name_not_allowed", - "only a database you created can be renamed. This one's name comes " - "from the module registry") - name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] - if not name: - raise err(400, "bad_request", "a database needs a name") - patch["name"] = name - if not patch: - raise err(400, "bad_request", "nothing to change") - - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} - for field, value in patch.items(): - if value is None: - entry.pop(field, None) # an explicit null CLEARS - else: - entry[field] = value - # An entry with nothing left in it is removed rather than stored empty: the read side - # skips empties anyway, and a bucket that accumulates `{}` per key is a document that - # grows forever and says nothing. - if entry: - data[key] = entry - else: - data.pop(key, None) - return data - - try: - session.runtime.update(_NAV_META_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the change was not saved: the store refused the write.") - return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} - - -@router.get("/nav/prefs") -def nav_prefs(session: Session = Depends(require_session)): - """This user's database-list folders, re-validated at serve time against what they may - currently see (a revoked page's placement vanishes with the page, and returns with it).""" - try: - stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} - except Exception: - stored = {} - keys, enumerated = _placeable_top_keys(session) - return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} - - -@router.post("/nav/prefs") -def save_nav_prefs(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" - keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: - data[session.uname] = clean - else: - data.pop(session.uname, None) - return data - - try: - session.runtime.update(_NAV_PREFS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") - return {"prefs": clean} - - -@router.get("/nav/schema/{key}") -def nav_schema(key: str, session: Session = Depends(require_session)): - """The database's schema drawer payload: its field contract + the semantic measures this - session may build with. Fail-closed on the SAME predicate as the nav — a key the session - may not open answers 403, never a redacted schema.""" - # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate - # (`may_open`) rather than the module grant machinery — `session.require` would 403 every - # ut key because a user table is deliberately not a module. - if key.startswith("ut_"): - import core.user_tables as user_tables - defn = user_tables.get(key, st=session.runtime) - if not defn or not user_tables.may_open(key, session.uname, session.admin, - st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a - # schema panel opened from it that says another is the drift a rename is supposed to - # remove, not create. - return {"key": key, - "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") - or defn.get("label") or key), - "source": defn.get("source") or "Blank", - "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], - "source": f.get("source") or "overlay", - "description": str(f.get("description") or "")} - for f in (defn.get("fields") or [])], - "measures": []} - session.require(key) - pages = perms.nav_pages(session.user) or [] - page = next((p for p in pages if p.get("key") == key), None) - if page is None: - raise err(404, "unknown_page", f"{key!r} is not a database this session can see") - fields = [] - if key in ("customer_data", "cohort", "customers"): - try: - import aios_grid - for f in aios_grid.FIELDS: - entry = {"key": f["key"], "label": f["label"], "type": f["type"], - "source": f["source"], - "description": str(f.get("description") or "")} - if f.get("options"): - entry["options"] = list(f["options"]) - fields.append(entry) - except Exception: - fields = [] - measures = [] - try: - from core import measure_resolve - team_id = perms.scope_team_id(session.user) - for m in measure_resolve.offer(team_id) or []: - measures.append({"key": str(m.get("key") or ""), - "label": str(m.get("label") or m.get("key") or ""), - "type": str(m.get("type") or "")}) - except Exception: - measures = [] - out = {"key": key, "label": page.get("label") or key, - "source": page.get("source") or "", "fields": fields, "measures": measures} - if not fields: - # Honest, never a mock: a database whose contract is not yet published says so. - out["note"] = "This database has not published a field contract yet." - return out +"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. + +SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who +may see what — a nav that hides a link the API would still serve is a UI courtesy, not a +permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), +so the nav and the 403 can never disagree about a grant. + +The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no +`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the +`prefs.json` Library preference is deliberately NOT applied — is documented on +`core.perms.nav_pages`, which is where it belongs: one place, both callers. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, perms, require_session + +router = APIRouter(prefix="/api/v1") + +#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder +#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a +#: surface (the server-filtered nav still decides what exists). +_NAV_PREFS_KEY = "nav_prefs" +_MAX_NAV_FOLDERS = 16 + +#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. +#: +#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a +#: separate bucket rather than another field in that one. Folders and placement are one +#: person's arrangement of their own rail — per-user by definition. What a database is CALLED +#: and what it looks like are facts about the database: a workspace where two people call the +#: same table different things has no shared vocabulary left to discuss it in. Same store, two +#: buckets, because they answer to two different owners. +#: +#: Shape: {"": {"icon": {"shape": , "tone": }, +#: "name": ""}} +_NAV_META_KEY = "nav_meta" + +#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these +#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, +#: so it is the one place the list is written twice. +#: +#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by +#: shape and maps the result, so a shape this whitelist let through and the renderer does not +#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the +#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of +#: that. If the two lists ever drift the symptom is an icon that silently reverts to the +#: default, which is the loudest SAFE failure available here. +_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", + "chart", "map", "users", "clock", "heart", "bolt"}) +_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) +_MAX_NAV_NAME = 60 + +#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. +#: +#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is +#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature +#: rather than a convenience one. +#: +#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. +#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An +#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one +#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls +#: off the list while forty slots hold repeat visits to the first. Keying by page makes +#: "recent" mean what the word means, and makes the cap bound the number of DATABASES +#: remembered rather than the number of clicks. +_NAV_RECENTS_KEY = "nav_recents" +_MAX_RECENTS = 50 + +#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a +#: precedent, not a preference. `user_tables.create` writes +#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses +#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 +#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — +#: with nothing to go red, because both ends are internally consistent. An integer instant has +#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect +#: shipped; this is that lesson applied before it.) +def _now() -> int: + return int(time.time()) + + +def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): + """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. + + `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown + or invisible key is dropped (it can return when the grant does — placement is cosmetic, + so pruning is loss-free). Unknown folder refs drop the placement, not the folder. + + `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the + `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than + pruned: keeping a placement whose database may since have been deleted is cosmetically + harmless, while pruning a live one is the silent data loss this function just stopped + causing. Never widened to non-`ut_` keys — those come from the compiled registry, which + cannot fail to enumerate. + """ + raw = raw if isinstance(raw, dict) else {} + folders, seen = [], set() + for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:40] + name = " ".join(str(f.get("name") or "").split())[:40] + if not fid or fid in seen or not name: + continue + seen.add(fid) + folders.append({"id": fid, "name": name}) + ids = {f["id"] for f in folders} + placement = {} + src = raw.get("placement") + if isinstance(src, dict): + for k, v in src.items(): + k, v = str(k)[:60], str(v)[:40] + if v not in ids: + continue # the folder is gone; the placement goes with it + if page_keys is None or k in page_keys: + placement[k] = v + elif keep_unknown_ut and k.startswith("ut_"): + placement[k] = v + return {"folders": folders, "placement": placement} + + +def _placeable_top_keys(session): + """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. + + ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by + construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and + `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's + user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. + So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement + naming one — on READ and on WRITE. + + The symptom was "a database I drag into a folder falls out of it later", never "it does not + work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy + (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. + The next reload served the pruned copy and the database was back at root. Built-in modules + (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly + why it read as intermittent rather than as a missing feature. + + ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents + block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the + ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 + fix did not reach. + + `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set + as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement + because the store blinked would be the original defect wearing a different cause. + """ + pages = perms.nav_pages(session.user) or [] + keys = {p["key"] for p in pages if not p.get("parent")} + try: + import core.user_tables as user_tables + # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never + # name a database this session cannot see, and the nav and this door agree by sharing + # one predicate rather than by two lists being kept in step. + # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is + # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires + # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured + # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the + # owner reports as "the Automation row arrives ten seconds late". + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + keys.add(str(e.get("key") or "")) + except Exception: + return keys, False + return keys, True + + +def _clean_icon(raw): + """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" + if not isinstance(raw, dict): + return None + shape, tone = raw.get("shape"), raw.get("tone") + if shape not in _ICON_SHAPES or tone not in _ICON_TONES: + return None + return {"shape": shape, "tone": tone} + + +def _read_nav_meta(runtime): + """The tenant's `nav_meta`, validated on the way OUT as well as in. + + Re-validating a read looks redundant and is not: the bucket outlives this code, an older + build may have written a shape this one no longer accepts, and the whitelist is mirrored + from a vocabulary that lives in another language. A row whose icon does not survive + validation renders the default mark — never a crash, and never a half-drawn glyph. + """ + try: + stored = runtime.get(_NAV_META_KEY) or {} + except Exception: + return {} # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return {} + out = {} + for key, meta in stored.items(): + if not isinstance(meta, dict): + continue + entry = {} + icon = _clean_icon(meta.get("icon")) + if icon: + entry["icon"] = icon + # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write + # route refuses a name for a built-in key, but a record written by an older build (or + # by hand) could still carry one — and honouring it would let the store rename + # `customer_data` on screen while `core/registry.py` and every other reader went on + # calling it Customer. A name that could not be written today is not served today. + name = meta.get("name") + if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): + entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] + if entry: + out[str(key)] = entry + return out + + +def _read_recents(runtime, uname, allowed): + """This user's recents, newest first, PRUNED to what they may currently see. + + ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: + + · the WRITE happens on every route open — it is the one hot path this file has — so it + must not build the whole nav to validate one key; + · a key whose grant was REVOKED must stop being offered without anyone running a + migration, and must come back if the grant does. That is `_clean_nav_prefs`' own + prune-never-invent posture, applied to a second bucket for the same reason. + + A key that is not in `allowed` therefore reaches the store and never reaches a screen — + which also means a stuffed key cannot be used to discover what exists: it comes back only + if the session could already see it. + """ + try: + stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} + except Exception: + return [] # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return [] + out = [] + for key, at in stored.items(): + key = str(key) + if allowed is not None and key not in allowed: + continue + try: + at = int(at) + except (TypeError, ValueError): + continue # a stamp this build cannot read is not a stamp + if at <= 0: + continue + out.append({"key": key, "at": at}) + out.sort(key=lambda r: r["at"], reverse=True) + return out[:_MAX_RECENTS] + + +# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── +# +# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the +# block that computed them (`_important_counts`) went with them. It read one store bucket PER +# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database +# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on +# **14** pages. It is **0 and 0** now. +# +# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the +# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold +# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. +# A budget that can be exhausted is a number that can be silently short, which is why it also had to +# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to +# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge +# instead of a late rail. Closes D-288 and D-289. +# +# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the +# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store +# read was — and duplicating them into the new route would have put two answers behind one badge. + +#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the +#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> +#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is +#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second +#: copy here would be free to drift from it ([[one-question-two-normalizers]]). +_WS_SUFFIX = "_table_workspace" + + +def _visible_views(doc, uname, is_admin, granted_ids): + """Every view on ONE database that THIS caller can see, from an already-read workspace doc. + + ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get + wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` + — one home per view, never both — so "how many views are marked important here" has a + DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and + `admin` has none, on the same database. A tenant-wide count would put a number in the owner's + rail that no view sidebar they can open would ever add up to. + + ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for + it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole + bucket per database (the N whole reads this block exists to avoid), and the only other option is + a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told + exists; a private import cannot. + """ + import core.table_store as table_store + out = {} + for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): + if isinstance(view, dict): + out[str(vid)] = view + for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): + if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): + out.setdefault(str(vid), view) + # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the + # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. + if granted_ids: + for stratum, blob in doc.items(): + if stratum == uname or not isinstance(blob, dict): + continue + for vid, view in (blob.get("views") or {}).items(): + if str(vid) in granted_ids and isinstance(view, dict): + out.setdefault(str(vid), view) + return out + + +def _view_record_count(cfg, cohorts): + """How many RECORDS this view resolves to, or None when that cannot be answered for free. + + ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact + because the view CARRIES its row set; the third — an ordinary filtered view — can only be + counted by running its filters over the records, and the records are the one thing this route + must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` + here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and + the database's `partial` flag says so, which is the whole of R6's second sentence applied to a + badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a + number that is short by an unknown amount. + + ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` + counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); + this end is the exact inverse — it has no rows at all and no window either. The two are honest + about different halves, which is why `partial` had to be on the wire rather than derived. + """ + if not isinstance(cfg, dict): + return None + # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` + # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + n = cohorts.get(lock) + return int(n) if isinstance(n, int) else None + # A curated row set carries its own count. + pids = cfg.get("memberPids") + if isinstance(pids, list) and pids: + return len(pids) + return None + + +def _visible_database_keys(session): + """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. + + ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: + + · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT + CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. + A star scan cannot borrow that: it would open the view bucket of a database this workspace's + catalogue does not include. `nav()` applies the catalogue filter; that helper never has. + · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so + it cannot be reached from another route without rebuilding the payload. + + So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same + order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the + tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes + the answer honest for a caller about to read a view bucket: a module surface with no workspace + (`sales`, `ar`) is not a database and has no views to star. + + ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` + excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: + a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even + though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious + reading — this ticket's first draft assumed the clause was excluding children and wrote a + docstring around a defect that does not exist. + + ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as + authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. + """ + import core.view_templates as view_templates + keys, enumerated = set(), True + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} + for p in (perms.nav_pages(session.user) or []): + key = str(p.get("key") or "") + if allowed is not None and key not in allowed \ + and str(p.get("parent") or "") not in allowed: + continue + if view_templates.workspace_key(key): + keys.add(key) + try: + import core.user_tables as user_tables + # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection + # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + k = str(e.get("key") or "") + if k: + keys.add(k) + except Exception: + enumerated = False + return keys, enumerated + + +def _granted_view_ids(session): + """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. + + ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above + takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with + the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to + answer "which views has this person starred", so the derivation moves here beside the reader + it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). + + ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming + me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to + agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test + would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so + it costs a dict lookup per id. + ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. + """ + import core.shares as shares + try: + return {str(v) for v in + ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) + .get("view") or []) + if shares.role_for("view", str(v), session.uname, + st=session.runtime) in ("view", "edit")} + except Exception: + return set() + + +@router.get("/nav") +def nav(session: Session = Depends(require_session)): + """`{pages: [{key,label,source?,chrome}], landing}`. + + Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you + are not allowed". A session that may open nothing at all is a misconfigured account, and it + gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is + how a permission bug hides in plain sight (X2's never-an-empty-200 rule). + """ + pages = list(perms.nav_pages(session.user) or []) + # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. + # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything + # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is + # its own databases, which is what "different set of databases at later waves" means. + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. + # + # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all + # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs + # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a + # row it removes and a row a store failure dropped are the SAME absence on the wire, and the + # client renders `null` for both — pixel-identical to still-loading. Naming the removals is + # what lets the shell tell "not part of this workspace" from "we could not read it". + _omitted = [] + if tmods != "all": + allowed = {str(k) for k in (tmods or [])} + _omitted = sorted({str(p.get("key")) for p in pages + if p.get("key") not in allowed + and (p.get("parent") or "") not in allowed}) + pages = [p for p in pages + if p.get("key") in allowed or (p.get("parent") or "") in allowed] + # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — + # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so + # the nav cannot offer a row the table routes would refuse. + # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. + # + # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one + # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below + # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms + # in-process** — the second figure is the one that matters, because with the network gone and + # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every + # other collapses to 20–161 ms. That is per-call CPU, and this is it. + # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ + # manage loop below consumes this same dict. + _ut_defs, _ut_locked_mode, _degraded = {}, "", [] + try: + import core.user_tables as user_tables + # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below + # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and + # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the + # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of + # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms + # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds + # apart because of CSS or ordering, they arrived when their route finished copying. + # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that + # needs a row, it raises with the reason instead of painting an empty grid. Use + # `all_tables` for that, and know you are buying the whole copy back. + _ut_defs = user_tables.all_defs(st=session.runtime) or {} + # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its + # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked + # the moment this import failed. A sentinel that can equal real data is not a sentinel. + _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) + _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) + # ⛔ AND THE EDITOR'S DENY REACHES THIS HALF TOO. `nav_entries` is `may_open`-filtered + # — creator, admin or a share — which knows nothing about the access toggle in + # Manage user. An administrator unticking a shared database left it on the rail. + # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. + import core.perm_scope as _perm_scope + # ⭐⭐ WAVE 37 · T07 (D-267, PRD amendment A8) — A RETIRED KEY CANNOT REACH THE NAV, WHATEVER + # THE TENANT DOCUMENT STILL CARRIES. + # + # W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` / + # `product_data` — ONE SUBJECT, ONE DATABASE — and `apply_plan` pops them from a tenant's + # document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live + # label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the + # PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the + # rebuild being scoped to `is_royal`). + # + # ⛔ SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN + # BLURRED: the stale ROW may still sit in a tenant's document, and removing it is + # `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document + # which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that + # migration has not reached them. A defect whose fix depends on a migration running is a + # defect that comes back on the one tenant the migration missed + # [[a-migration-that-runs-on-the-next-write]]. + # + # ⚠ `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift + # this guard exists to prevent, and it would go stale the first time a third key is retired. + # ⚠ Inside the same `try:` as the rest of the user-table read on purpose — if the import + # fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A + # guard that silently stops guarding is worse than one that says it could not run. + try: + from odoo_relational import RETIRED_KEYS as _retired + except Exception: + _retired = () + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=_lent): + if e["key"] in _retired: + continue + if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): + continue + pages.append({"key": e["key"], "label": e["label"], + "source": e.get("source") or "Blank", "chrome": "main"}) + except Exception: + # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. + # + # A store blip must not take the whole nav down with it, and that half stands. What + # changed is that it used to answer **200 OK with every database missing** and say + # nothing — so a client cannot tell "this tenant has no databases" from "we could not + # read them", and the rail renders the same complete-looking thing either way. That is + # the owner's *"Connectors and Automation module still disappears"* class of report: + # the payload is a claim about what exists, and a claim it could not verify has to be + # marked as such. `degraded` is that mark; the shell renders it in the affected slot. + _degraded.append("databases") + pass + if not pages: + if tmods != "all": + # A provisioned tenant with no modules and no databases YET is a legitimate empty + # state, not a misconfigured account — the client renders "create your first + # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is + # empty rather than leaving 200-[] ambiguous. + return {"pages": [], "landing": None, "empty": "no_databases"} + raise err(403, "no_surfaces", + "your account has no dashboards assigned. Ask an administrator.") + # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry + # rows, after the tenant module filter, after the user tables. Merged HERE rather than + # applied by the client for one reason: `label` is what every reader of this payload shows, + # including the Settings modal's `moduleLabels` map, so a client-side merge would put the + # override in one door and the registry label in the other. + # + # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A + # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If + # either ever starts handing back cached or module-level dicts, this loop would write one + # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant + # leak with no symptom until two tenants rename the same registry key. Copy the rows before + # merging on the day that invariant changes. + meta = _read_nav_meta(session.runtime) + # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. + # + # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS + # ORDER so a cold container would reach the databases somebody actually works in first. Every + # one of those was a correct answer to the wrong question: the block was `N` per-database store + # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block + # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per + # `/nav`** before this ticket, **0 and 0** after. + # + # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer + # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. + # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave + # 23), and it was only computed this early so the deleted budget could spend itself in its order. + _page_keys = {str(p.get("key", "")) for p in pages} + recents = _read_recents(session.runtime, session.uname, _page_keys) + # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. + # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. + # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, + # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. + # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an + # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. + for p in pages: + # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because + # the server is the only end that knows — the client cannot see who created a user + # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control + # only where the write would actually land, and the route re-checks it regardless. + # + # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the + # de5037f share-grant admission widened `may_open`, so a row's presence now includes + # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are + # therefore answered from the DEFINITION: creator-or-admin strictly, matching the + # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no + # controls — the honest shape (the old code offered rename to users the route 403'd). + key = str(p.get("key", "")) + if key.startswith("ut_"): + _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") + _mine = bool(session.admin or (_creator and _creator == session.uname)) + p["manage"] = _mine + p["canDelete"] = _mine + # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. + # + # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md + # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still + # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the + # live example; their rows arrive from the engine, so a "+" row there could only + # refuse, which R8 calls a fake affordance. + # + # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is + # inlined only because `_ut_defs` is already in hand — calling the predicate per row + # would be one store read per database on every nav request — and the VALUE comes + # from the module's own constant rather than a copied string, so the two cannot drift + # to different answers. If that predicate ever grows a second condition, this must + # become a call. + # + # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is + # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while + # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs + # a missing hint, never a write that should not have landed. + if (_ut_locked_mode + and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): + p["locked"] = True + elif session.admin: + p["manage"] = True + # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database + # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer + # never had to test for the key. R4 retires the count from the rail and the flyout entirely + # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. + # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an + # absent key reads as `undefined` there, which is the same thing the optional field already + # meant for a non-database row. Flagged to A rather than assumed harmless. + entry = meta.get(p.get("key")) if meta else None + if not entry: + continue + if entry.get("icon"): + p["icon"] = entry["icon"] + if entry.get("name"): + p["label"] = entry["name"] + landing = perms.landing_page(session.user) + if landing and not any(p.get("key") == landing for p in pages): + landing = pages[0].get("key") + # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks + # for. A second round trip for a list this short, computed from a bucket this route is + # already holding the store open for, would be a request per page load for no gain. + # + # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases + # included. Pruning against `perms.nav_pages` alone would silently drop every user database + # from Home's recents: the exact surface R7 is about, invisible, with every gate green. + # + # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — + # had the identical bug and nobody connected the two for four waves. It is fixed at the + # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so + # both doors finally agree on what a placeable key is. This block keeps using the assembled + # list because it already holds it: re-enumerating here would be a second store read for an + # answer sitting in a local variable. + # + # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, + # because the important-count block spends its budget in RECENTS ORDER (see there). It is still + # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here + # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. + # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. + # `omitted` — this workspace's catalogue does not include these modules. Deliberate. + # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says + # so in the affected slot instead of rendering a confident nothing. + # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as + # `[]` rather than being omitted when empty, which is the same rule `limits` follows. + return {"pages": pages, "landing": landing, "recents": recents, + "omitted": _omitted, "degraded": _degraded} + + +@router.post("/nav/opened") +def nav_opened(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. + + The client calls this on every route commit, so this is the only write in this file on a + hot path, and three things follow from that: + + · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per + page open against an HF-Dataset-backed store would put a network round trip inside every + navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; + this is. + · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string + would make the stamp cost more than the navigation that triggered it — and it would buy + nothing, because the READ prunes to what the session may currently see. An unknown or + revoked key is stored and never served back. + · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client + grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to + go, which is the same rule the read applies. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip()[:60] + if not key: + raise err(400, "bad_request", "no page was named") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was recorded.") + stamp, uname = _now(), session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + mine[key] = stamp + if len(mine) > _MAX_RECENTS: + # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape + # sorts as 0 and is the first thing evicted, which is the right answer for a value + # this route can no longer read. + def _at(item): + try: + return int(item[1]) + except (TypeError, ValueError): + return 0 + mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) + data[uname] = mine + return data + + try: + session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was recorded.") + return {"key": key, "at": stamp} + + +@router.post("/nav/meta") +def save_nav_meta(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. + + A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the + asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so + replacing the document whole is what makes a deleted folder stay deleted. This bucket is + shared by every admin in the tenant: a wholesale write here means whoever saves last + silently erases what the other one named while their tab was open. + + THREE WALLS, all fail-closed: + · the SESSION must be able to open the key at all (the same predicate the nav uses, so + the rail and this route cannot disagree about what exists); + · the WRITE is admin-only — renaming a database is a change every user in the tenant + sees, which is the definition of an administrative act here; + · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry + literals: honouring an override would leave this payload and `core/registry.py` + calling the same module two different things, and the wave-16 lesson is that the + client must not be the place that decides what a payload meant. + + `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an + icon change two independent writes rather than a race between them. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip() + if not key: + raise err(400, "bad_request", "no database was named") + # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── + # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. + # + # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what + # `user_tables.may_open` already means. A database you made is yours to name; requiring + # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` + # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, + # because a user table is not a module. Same split `/nav/schema/{key}` makes. + # · everything else — admin only, and icon only. There is no owner of `customer_data` to + # defer to, and its label is a compiled registry literal (refused below regardless). + if key.startswith("ut_"): + import core.user_tables as user_tables + if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( + key, session.uname, session.admin, st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + else: + if not session.admin: + raise err(403, "forbidden", + "only an administrator can change a built-in database's icon") + session.require(key) + + patch = {} + if "icon" in body: + icon = _clean_icon(body.get("icon")) + if body.get("icon") is not None and icon is None: + # Loud, not silent. A shape this build does not know is a CLIENT that has drifted + # from this whitelist, and answering 200 to a write that stored nothing is how + # that drift stays invisible until a user reports "my icon keeps resetting". + raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") + patch["icon"] = icon + if "name" in body: + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database you created can be renamed. This one's name comes " + "from the module registry") + name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] + if not name: + raise err(400, "bad_request", "a database needs a name") + patch["name"] = name + if not patch: + raise err(400, "bad_request", "nothing to change") + + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + for field, value in patch.items(): + if value is None: + entry.pop(field, None) # an explicit null CLEARS + else: + entry[field] = value + # An entry with nothing left in it is removed rather than stored empty: the read side + # skips empties anyway, and a bucket that accumulates `{}` per key is a document that + # grows forever and says nothing. + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + session.runtime.update(_NAV_META_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} + + +@router.get("/nav/prefs") +def nav_prefs(session: Session = Depends(require_session)): + """This user's database-list folders, re-validated at serve time against what they may + currently see (a revoked page's placement vanishes with the page, and returns with it).""" + try: + stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} + except Exception: + stored = {} + keys, enumerated = _placeable_top_keys(session) + return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} + + +@router.post("/nav/prefs") +def save_nav_prefs(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Wholesale replace, like the table folder stratum — the client sent its complete + picture, validated here; a partial merge would resurrect deleted folders forever.""" + keys, enumerated = _placeable_top_keys(session) + clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + if clean["folders"] or clean["placement"]: + data[session.uname] = clean + else: + data.pop(session.uname, None) + return data + + try: + session.runtime.update(_NAV_PREFS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the folder change was not saved: the store refused the write.") + return {"prefs": clean} + + +@router.get("/nav/schema/{key}") +def nav_schema(key: str, session: Session = Depends(require_session)): + """The database's schema drawer payload: its field contract + the semantic measures this + session may build with. Fail-closed on the SAME predicate as the nav — a key the session + may not open answers 403, never a redacted schema.""" + # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate + # (`may_open`) rather than the module grant machinery — `session.require` would 403 every + # ut key because a user table is deliberately not a module. + if key.startswith("ut_"): + import core.user_tables as user_tables + defn = user_tables.get(key, st=session.runtime) + if not defn or not user_tables.may_open(key, session.uname, session.admin, + st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a + # schema panel opened from it that says another is the drift a rename is supposed to + # remove, not create. + return {"key": key, + "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") + or defn.get("label") or key), + "source": defn.get("source") or "Blank", + "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], + "source": f.get("source") or "overlay", + "description": str(f.get("description") or "")} + for f in (defn.get("fields") or [])], + "measures": []} + session.require(key) + pages = perms.nav_pages(session.user) or [] + page = next((p for p in pages if p.get("key") == key), None) + if page is None: + raise err(404, "unknown_page", f"{key!r} is not a database this session can see") + fields = [] + if key in ("customer_data", "cohort", "customers"): + try: + import aios_grid + for f in aios_grid.FIELDS: + entry = {"key": f["key"], "label": f["label"], "type": f["type"], + "source": f["source"], + "description": str(f.get("description") or "")} + if f.get("options"): + entry["options"] = list(f["options"]) + fields.append(entry) + except Exception: + fields = [] + measures = [] + try: + from core import measure_resolve + team_id = perms.scope_team_id(session.user) + for m in measure_resolve.offer(team_id) or []: + measures.append({"key": str(m.get("key") or ""), + "label": str(m.get("label") or m.get("key") or ""), + "type": str(m.get("type") or "")}) + except Exception: + measures = [] + out = {"key": key, "label": page.get("label") or key, + "source": page.get("source") or "", "fields": fields, "measures": measures} + if not fields: + # Honest, never a mock: a database whose contract is not yet published says so. + out["note"] = "This database has not published a field contract yet." + return out diff --git a/api/routes_products.py b/api/routes_products.py index 5eb6bdf176bb4db418309fad9f635718262287b2..968e162249a92d28480cae1147d1e416c65bc767 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -115,15 +115,190 @@ def scoped_pool(session: Session): return pids, team_id, rows_src, fields_base +#: The semantic ENTITY topic this grid IS. `model/topics/odoo_products.yml` names the same store +#: key in its `grid:` field, so the binding is stated at both ends rather than inferred. +TOPIC = "product_data" +SEM_TOPIC = "odoo_products" + + +def _measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: + pass + + +def _measure_stamp(rt, team_id): + """The cached pool's build timestamp — the DATA STAMP in every memo key, so a pool refresh + invalidates memoised measure answers exactly when the underlying rows changed. Same + discipline as `routes_customers._pool_stamp`, over THIS route's cache key.""" + entry = rt.pool_cache.get(("product_pool", team_id)) + return entry[0] if isinstance(entry, tuple) and entry else 0 + + +def product_measures(): + """The measure OFFER for this database, or `[]` when the semantic layer cannot answer. + + ⭐⭐ W37-T10 / owner item 4 (R1) — this is what turns `measures: []` into a real catalogue. + Owner: *"Odoo products should have a lookbsck metrics like sales etc."* The list is DERIVED + from `model/topics/odoo_products.yml`'s `measures:` binding, never written here — a second + list would be a second definition of the same fact. + + ⚠ `[]` ON FAILURE IS THE CORRECT DEGRADE and it is not silent: `entity_measures` refuses a key + it cannot prove, the reasons are readable through `semantic.entity_measure_refusals`, and the + client's own rule (`offerMeasure={measures.length > 0}`) then hides the Metric kind rather + than offering a column that could only render blank. + """ + try: + from harness import semantic as sem + return sem.entity_measures(SEM_TOPIC) + except Exception as e: # noqa: BLE001 + _measure_err("product-measure-offer", e) + return [] + + +def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer): + """`{pid: {field key: value}}` for every measure column on this grid. + + ⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's + `offerMeasure` on and lets a user mint a Metric column that would then be blank forever — + this repo's most repeated defect ([[reachable-is-not-the-same-as-built]]). Nothing else + computes product-grain measure values: `core.measure_resolve.column_values` is CUSTOMER-grain + (it resolves through `harness.measure_filter`, whose entity is the partner), which is exactly + what this route's docstring used to record as an honest absence. + + ⭐ ONE QUERY PER WINDOW, not per column. Two Metric columns over the same 90 days are one + grouped scan; the fan-out to field keys happens in memory afterwards. + + ⛔ THE JOIN IS BY SKU CODE, per contract C1's trap. `sales_lines.product_code` reproduces this + pool's identity (`default_code`, or `pid:` for the 33 codeless actives) and `pid` is + a CRC32 of it — so the map is `row['code'] -> row['pid']`, taken off the rows we already hold. + Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries. + """ + import aios_grid + + mfields = aios_grid.measure_fields_of(fields) + if not mfields or not offer: + return {} + from harness import semantic as sem + from harness import windows as _wn + + by_key = {m["key"]: m for m in offer} + codes = {r["pid"]: r["code"] for r in rows_src if r.get("pid") is not None} + + # Group the columns by WINDOW: same window, same scan. + by_window = {} + for f in mfields: + spec = f.get("measure") or {} + if spec.get("key") not in by_key: + continue # not offered for this caller -> the column stays blank, honestly + w = _wn.normalize(spec.get("window")) + rng = _wn.resolve(spec.get("window"), today) if w is not None else None + if rng is None: + # ⛔ NEVER WIDEN TO ALL TIME. An unresolvable window is an unanswerable column, and + # silently answering a different question is worse than a blank one. + continue + by_window.setdefault((rng[0], rng[1]), []).append(f) + + out = {} + for (dfrom, dto), group in by_window.items(): + keys = tuple(sorted({f["measure"]["key"] for f in group})) + mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys) + if mkey not in memo: + try: + memo[mkey] = sem.entity_measure_values( + SEM_TOPIC, list(keys), date_from=dfrom, date_to=dto, team_id=team_id, + # ⛔ SERVICES ARE IN THIS CATALOGUE (37 active), so they must NOT be excluded + # here. The service filter exists to stop Delivery Charges polluting a SKU + # RANKING; on a per-row column the row IS the service product, and excluding + # it would print a FALSE 0 — which under C1's rule asserts "sold nothing". + exclude_services=False, offer=offer) + except Exception as e: # noqa: BLE001 + _measure_err("product-measure-column", e) + # Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure + # memoises, a TRANSIENT one (store still warming after a restart) must not, or + # the cells stay blank for the rest of the session long after the data landed. + transient = False + try: + from harness import datastore as _ds + transient = not _ds.ready() + except Exception: # noqa: BLE001 + transient = False + if transient: + continue + memo[mkey] = None + vals = memo[mkey] + if vals is None: + continue + # C1's empty-window rule, applied to EVERY row — not only the ones with a group. 72% of + # this catalogue has no group in a 90-day window, so this loop IS the common case. + filled = {pid: sem.entity_zero_fill(vals.get(code), list(keys), offer) + for pid, code in codes.items()} + for f in group: + k, fkey = f["measure"]["key"], f["key"] + for pid, cell in filled.items(): + if k in cell: + out.setdefault(pid, {})[fkey] = cell[k] + if len(memo) > 100: # a lifetime cache, not a leak + memo.clear() + return out + + +def _measure_condition_sets(views, offer, rows_src, team_id, today): + """`{rule id: [pid, …]}` for every measure CONDITION across this session's saved views. + + ⚠ THE ANSWER IS KEYED BY pid, NOT BY SKU CODE. `entity_measure_sets` answers in the SOURCE + dim's own values (the code) because it cannot know how a grid hashes identity; the map back + is this route's job, exactly as it is for the column cells. + """ + if not offer: + return {} + from harness import semantic as sem + + admitted = {m["key"] for m in offer} + by_id = {} + for v in views or []: + cfg = (v.get("config") or {}) + for rule in sem.entity_measure_leaves(cfg.get("filters"), admitted): + rid = str(rule.get("id") or "") + if rid: + by_id[rid] = rule # last statement of the question wins + if not by_id: + return {} + pid_by_code = {r["code"]: r["pid"] for r in rows_src if r.get("pid") is not None} + try: + sets = sem.entity_measure_sets( + SEM_TOPIC, list(by_id.values()), today, team_id=team_id, + keys_by_id=list(pid_by_code), offer=offer, exclude_services=False) + except Exception as e: # noqa: BLE001 + _measure_err("product-measure-sets", e) + return {} + # A code the pool does not carry is dropped rather than keyed on None — the same rule the + # cell path applies, for the same reason. + return {rid: sorted(pid_by_code[c] for c in codes if c in pid_by_code) + for rid, codes in sets.items()} + + def product_assembly(session: Session, scope: str = "product", storage_key: str = "", consume_corrections: bool = True): """The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so the /workspace and events routes consume either interchangeably. - One deliberate absence, a topic fact rather than a gap: - * `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is CUSTOMER-grain (the - C-TOPIC v1 descope, booked in the wave doc); the events ctx therefore refuses measure - creates on this surface, which is the correct fail-closed shape. + ⭐⭐ W37-T10 / OWNER ITEM 4 (R1) — `measures` IS NO LONGER EMPTY, and the header above it used + to record the opposite as a topic fact. It was one: `core.measure_resolve` is CUSTOMER-grain, + so nothing here could resolve a product-grain measure. What changed is that the ENTITY path + exists now (`semantic.entity_measures` / `entity_measure_values`, bound in + `model/topics/odoo_products.yml`), so this surface serves a real six-metric catalogue AND the + cells to go with it — the offer and the resolver landing together, deliberately, because an + offer whose values nobody computes is a column that renders blank forever. + + One deliberate absence remains, and it is a real one: + * `measure_sets` stays EMPTY — that is the measure CONDITION channel (a filter answered + server-side as a pid set), which still runs through the customer-grain + `measure_resolve.condition_sets`. So a Metric COLUMN works here; a Metric FILTER does + not yet. Stated rather than left to be discovered: `clean_filter_tree` drops a measure + condition whose key is not in `measure_keys`, so the refusal is already fail-closed. ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. Wave 16 passed `with_cohorts=False` because there was one customer-keyed cohort bucket and product pids are CRC32 hashes of SKU codes; @@ -172,12 +347,29 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str fields = [f for f in fields if f.get("key") not in hidden] rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] + today = time.strftime("%Y-%m-%d") + stamp = _measure_stamp(session.runtime, team_id) + measures = product_measures() + + # R9: the Cohorts column's cells, built from THIS topic's lists. ⭐ W37-T10 — the measure + # half of this same read-only channel is no longer empty; the customer assembly merges its + # measure columns here for exactly the same reason and through the same door. + derived = aios_grid.cohort_cells(lists) + for pid, cells in _measure_cells(fields, rows_src, team_id, today, stamp, + session.runtime.measure_memo, measures).items(): + derived.setdefault(pid, {}).update(cells) + + # ⛔ THE CONDITION CHANNEL SHIPS WITH THE OFFER, NOT AFTER IT. `routes_grid` derives + # `measure_keys` from this same list, so the moment `measures` is non-empty the client's + # filter builder offers measure conditions — and an id missing from `measureSets` renders as + # PENDING ("Calculating…") and matches nothing, permanently. Serving the offer without this + # would trade one honest absence for a spinner that never resolves. + measure_sets = _measure_condition_sets(views, measures, rows_src, team_id, today) + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, - # R9: the Cohorts column's cells, built from THIS topic's lists. Same read-only - # `derived` channel the customer assembly uses — the measure half stays empty. - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + "derived": derived, + "measures": measures, "measure_sets": measure_sets, "today": today, "team_id": team_id} diff --git a/api/routes_script_views.py b/api/routes_script_views.py index 3481df79475146575d2a5ae42ed2b05708dfd4c4..ad92052fa5a3852e6bf93d1d1f6f2dbe8e7bc02c 100644 --- a/api/routes_script_views.py +++ b/api/routes_script_views.py @@ -1,329 +1,439 @@ -"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). - -Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user -can build whatever they want through the Agent chat interface. be able to create any dashboard -they want. User should have the ability to see the code AND the dashboard output of course… Limit -the code script View per database… Any agent can add into more AI script, so we can see different -versions or different things the AI code for us."* - - GET /api/v1/script-views?database=K the views bound to ONE database - POST /api/v1/script-views create one {database, name?, source} - GET /api/v1/script-views/{id} one view, its source and its history - PUT /api/v1/script-views/{id} a NEW VERSION of the source - DELETE /api/v1/script-views/{id} drop it - POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} - -⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may -read ONLY the database it lives in, and a script that names another database is REFUSED with a -message naming both. *Not capped*: there is **no limit on how many script views a database may -carry**, because that is how an agent offers three attempts and the owner picks one. So nothing -below counts views. What IS bounded is what makes them big — one source is capped, and one view's -edit history is capped and REPORTS what it dropped. - -⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed -`session.user`, so a script written by an administrator and opened by a scoped analyst reads the -ANALYST's rows. The author decides what the code does; the reader decides what it can see. -⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script -that exfiltrates anything, because the only thing a script can return is a render spec drawn on -the screen of the person who ran it. There is no network, no file and no second reader. - -⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a -coroutine that would block the event loop for every other request in the container. FastAPI runs a -plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. -""" -import threading -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. -VIEWS_KEY = "script_views" - -MAX_NAME = 80 -MAX_SOURCE_BYTES = 128 * 1024 - -#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far -#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader -#: can see the history is partial instead of concluding the view was only ever saved twice. -MAX_HISTORY = 40 - -#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal -#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, -#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what -#: gives up. A 429 that says so is honest; an unbounded fork is not. -MAX_CONCURRENT_RUNS = 4 -_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this database has no script views", never to a 500 on the view rail.""" - try: - found = rt.get(VIEWS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _database_ok(session, database): - """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. - - ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True - for an ADMIN on any key at all, including one no database answers to. So a create validated - with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing - can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it - RAISES `UnknownTable`, which is exactly the question being asked. - """ - import core.perm_scope as perm_scope - try: - perm_scope.scoped_fields(session.user, database, st=session.runtime) - except perm_scope.UnknownTable: - raise err(404, "no_database", f"there is no database '{database}' in this workspace") - except perm_scope.Denied: - raise err(403, "forbidden", f"your account may not read '{database}'") - except perm_scope.Unresolvable as exc: - # The database is real and cannot be served under this call's constraints. Standing rule - # 1's second sentence: report the cause and the recommendation, never a bare refusal. - raise err(409, "unresolvable", str(exc)) from None - - -def _clean_source(raw): - source = str(raw or "") - if not source.strip(): - raise err(400, "no_source", "a script view needs some code") - if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - raise err(413, "source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") - return source - - -def _row(rec, *, source=False): - """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" - out = {"id": rec.get("id") or "", "database": rec.get("database") or "", - "name": rec.get("name") or "", "author": rec.get("author") or "", - "version": int(rec.get("version") or 1), - "created": rec.get("created") or "", "updated": rec.get("updated") or "", - "versions": len(rec.get("history") or []) + 1, - "trimmed": int(rec.get("trimmed") or 0)} - if source: - out["source"] = rec.get("source") or "" - out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", - "created": h.get("created") or "", - "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} - for h in reversed(rec.get("history") or []) if isinstance(h, dict)] - return out - - -def _limits(): - return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, - "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, - # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. - "maxViewsPerDatabase": None} - - -def _put(session, view_id, mutate): - """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" - def _set(cur): - cur = dict(cur or {}) - nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) - if nxt is None: - cur.pop(view_id, None) - else: - cur[view_id] = nxt - return cur - - session.runtime.update(VIEWS_KEY, _set, flush="sync") - - -def _mine_or_admin(session, rec): - """Who may EDIT or DELETE a view: its author, or an administrator. - - ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database - may run any view on it, under their OWN scope. That is the whole of "so we can see different - versions or different things the AI code for us": a colleague's attempt is worth nothing if - only its author can open it. - """ - if session.admin or str(rec.get("author") or "") == session.uname: - return - raise err(403, "forbidden", "only the author or an administrator can change this script view") - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/script-views") -def list_script_views(database: str = "", session: Session = Depends(require_session)): - """Every script view bound to ONE database, newest first. `database` is required.""" - key = str(database or "").strip() - if not key: - raise err(400, "no_database", "name the database whose script views you want") - _database_ok(session, key) - rows = [_row(rec) for rec in _all(session.runtime).values() - if isinstance(rec, dict) and str(rec.get("database") or "") == key] - rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) - return {"database": key, "views": rows, "limits": _limits()} - - -@router.post("/script-views") -def create_script_view(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. - - A script view that cannot be run is a broken feature the reader discovers by pressing a button, - and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, - so the refusal arrives while the author still has the code in front of them. - """ - import secrets # noqa: PLC0415 - import core.script_sandbox as sandbox # noqa: PLC0415 - - body = body if isinstance(body, dict) else {} - database = str(body.get("database") or "").strip() - if not database: - raise err(400, "no_database", "a script view is bound to one database") - _database_ok(session, database) - source = _clean_source(body.get("source")) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - view_id = "sv_" + secrets.token_urlsafe(9) - rec = {"id": view_id, "database": database, - "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], - "source": source, "author": session.uname, "version": 1, - "created": _now(), "updated": _now(), "history": [], "trimmed": 0} - _put(session, view_id, lambda _prior: rec) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell the author their - # script was saved when it was not. - raise err(503, "store_unavailable", "the script view was NOT created") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.get("/script-views/{view_id}") -def get_script_view(view_id: str, session: Session = Depends(require_session)): - """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _database_ok(session, str(rec.get("database") or "")) - return {"view": _row(rec, source=True), "limits": _limits()} - - -@router.put("/script-views/{view_id}") -def update_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - source = _clean_source(body.get("source")) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - def _mutate(prior): - prior = dict(prior or rec) - history = list(prior.get("history") or []) - history.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", "created": prior.get("updated") or ""}) - dropped = max(0, len(history) - MAX_HISTORY) - prior["history"] = history[dropped:] if dropped else history - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - if body.get("name"): - prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the new version was NOT saved") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.delete("/script-views/{view_id}") -def delete_script_view(view_id: str, session: Session = Depends(require_session)): - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - _put(session, view_id, lambda _prior: None) - return {"deleted": view_id} - - -@router.post("/script-views/{view_id}/run") -def run_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. - - ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser - executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key - before this function ever sees it, so a renderer cannot be talked into running something by a - script that was itself perfectly well behaved. - - ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess - wait on the event loop is a ten-second outage for the whole container. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - database = str(rec.get("database") or "") - # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. - # It is checked exactly as a stored one is, because "unsaved" is not a permission. - draft = (body or {}).get("source") if isinstance(body, dict) else None - source = _clean_source(draft) if draft else str(rec.get("source") or "") - # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and - # asking twice builds a registry topic's pool twice. The three refusals are translated below - # instead, which is the same wall reached through the same door. - - if not _RUN_SLOTS.acquire(blocking=False): - raise err(429, "busy", - f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " - f"Try again in a moment") - try: - out = sandbox.run_view(session.user, database, source, st=session.runtime) - finally: - _RUN_SLOTS.release() - - # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" - # answered 200 would be a permission decision the client has to go looking for, and every - # other door in this app answers 403 for it. Everything BELOW this line is a well-formed - # request whose ANSWER is that the script did not produce a view — that is a 200 with - # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. - if out.get("code") == "unknown_table": - raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") - if out.get("code") == "denied": - raise err(403, "forbidden", out.get("message") or "your account may not read that database") - if out.get("code") == "unresolvable": - refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") - refusal.detail["error"]["limit"] = out.get("limit") or {} - raise refusal - - answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), - "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), - "ms": int(out.get("ms") or 0), "code": out.get("code") or "", - # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits - # were applied; on a Windows host the memory and CPU ones were not, and a screen - # that claims an enforcement which did not happen is the failure the rule is about. - "caps": out.get("caps") or {}} - if not answer["ok"]: - answer["error"] = out.get("message") or "the script view did not produce a view" - return answer +"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). + +Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user +can build whatever they want through the Agent chat interface. be able to create any dashboard +they want. User should have the ability to see the code AND the dashboard output of course… Limit +the code script View per database… Any agent can add into more AI script, so we can see different +versions or different things the AI code for us."* + + GET /api/v1/script-views?database=K the views bound to ONE database + POST /api/v1/script-views create one {database, name?, source} + GET /api/v1/script-views/{id} one view, its source and its history + PUT /api/v1/script-views/{id} a NEW VERSION of the source + DELETE /api/v1/script-views/{id} drop it + POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} + POST /api/v1/script-views/{id}/revert go back to an earlier version {version} + +⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may +read ONLY the database it lives in, and a script that names another database is REFUSED with a +message naming both. *Not capped*: there is **no limit on how many script views a database may +carry**, because that is how an agent offers three attempts and the owner picks one. So nothing +below counts views. What IS bounded is what makes them big — one source is capped, and one view's +edit history is capped and REPORTS what it dropped. + +⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed +`session.user`, so a script written by an administrator and opened by a scoped analyst reads the +ANALYST's rows. The author decides what the code does; the reader decides what it can see. +⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script +that exfiltrates anything, because the only thing a script can return is a render spec drawn on +the screen of the person who ran it. There is no network, no file and no second reader. + +⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a +coroutine that would block the event loop for every other request in the container. FastAPI runs a +plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. +""" +import threading +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. +VIEWS_KEY = "script_views" + +MAX_NAME = 80 +MAX_SOURCE_BYTES = 128 * 1024 + +#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far +#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader +#: can see the history is partial instead of concluding the view was only ever saved twice. +MAX_HISTORY = 40 + +#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal +#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, +#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what +#: gives up. A 429 that says so is honest; an unbounded fork is not. +MAX_CONCURRENT_RUNS = 4 +_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this database has no script views", never to a 500 on the view rail.""" + try: + found = rt.get(VIEWS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _database_ok(session, database): + """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. + + ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True + for an ADMIN on any key at all, including one no database answers to. So a create validated + with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing + can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it + RAISES `UnknownTable`, which is exactly the question being asked. + """ + import core.perm_scope as perm_scope + try: + perm_scope.scoped_fields(session.user, database, st=session.runtime) + except perm_scope.UnknownTable: + raise err(404, "no_database", f"there is no database '{database}' in this workspace") + except perm_scope.Denied: + raise err(403, "forbidden", f"your account may not read '{database}'") + except perm_scope.Unresolvable as exc: + # The database is real and cannot be served under this call's constraints. Standing rule + # 1's second sentence: report the cause and the recommendation, never a bare refusal. + raise err(409, "unresolvable", str(exc)) from None + + +def _clean_source(raw, *, allow_empty=False): + """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. + + ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. + Picking "Custom View" in the mode picker mints the view immediately, before a line of code + exists, so that `mode === 'script'` always implies a real id and no reader has to carry a + "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). + A view that has been created and not yet written is therefore a REAL, legible state: the editor + shows its starter placeholder and the Run control is right there. + A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a + working script by saving nothing over it is not an edit anybody means to make. So the guard + stays exactly where it was on that door, and a person who wants the view gone deletes it. + + ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns + None, so "an empty script is storable" looked true and was written into a mailbox answer another + lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox + knows nothing about. Two validators for one question, disagreeing + [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no + unit of this feature was asserting. + """ + source = str(raw or "") + if not source.strip() and not allow_empty: + raise err(400, "no_source", "a script view needs some code") + if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + raise err(413, "source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") + return source + + +def _row(rec, *, source=False): + """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" + out = {"id": rec.get("id") or "", "database": rec.get("database") or "", + "name": rec.get("name") or "", "author": rec.get("author") or "", + "version": int(rec.get("version") or 1), + "created": rec.get("created") or "", "updated": rec.get("updated") or "", + "versions": len(rec.get("history") or []) + 1, + "trimmed": int(rec.get("trimmed") or 0), + # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the + # row (not only the history) because the editor's header is where a person reads "what + # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back + # legible as an event rather than as a coincidence of matching code. + "restoredFrom": rec.get("restoredFrom")} + if source: + out["source"] = rec.get("source") or "" + out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", + "created": h.get("created") or "", + "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} + for h in reversed(rec.get("history") or []) if isinstance(h, dict)] + return out + + +def _limits(): + return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, + "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, + # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. + "maxViewsPerDatabase": None} + + +def _put(session, view_id, mutate): + """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" + def _set(cur): + cur = dict(cur or {}) + nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) + if nxt is None: + cur.pop(view_id, None) + else: + cur[view_id] = nxt + return cur + + session.runtime.update(VIEWS_KEY, _set, flush="sync") + + +def _mine_or_admin(session, rec): + """Who may EDIT or DELETE a view: its author, or an administrator. + + ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database + may run any view on it, under their OWN scope. That is the whole of "so we can see different + versions or different things the AI code for us": a colleague's attempt is worth nothing if + only its author can open it. + """ + if session.admin or str(rec.get("author") or "") == session.uname: + return + raise err(403, "forbidden", "only the author or an administrator can change this script view") + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/script-views") +def list_script_views(database: str = "", session: Session = Depends(require_session)): + """Every script view bound to ONE database, newest first. `database` is required.""" + key = str(database or "").strip() + if not key: + raise err(400, "no_database", "name the database whose script views you want") + _database_ok(session, key) + rows = [_row(rec) for rec in _all(session.runtime).values() + if isinstance(rec, dict) and str(rec.get("database") or "") == key] + rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) + return {"database": key, "views": rows, "limits": _limits()} + + +@router.post("/script-views") +def create_script_view(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. + + A script view that cannot be run is a broken feature the reader discovers by pressing a button, + and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, + so the refusal arrives while the author still has the code in front of them. + """ + import secrets # noqa: PLC0415 + import core.script_sandbox as sandbox # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "no_database", "a script view is bound to one database") + _database_ok(session, database) + # ⭐ is CREATE's alone - see . The picker mints a view the + # moment a person chooses the mode, so the empty source is the normal first state, not a slip. + source = _clean_source(body.get("source"), allow_empty=True) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + view_id = "sv_" + secrets.token_urlsafe(9) + rec = {"id": view_id, "database": database, + "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], + "source": source, "author": session.uname, "version": 1, + "created": _now(), "updated": _now(), "history": [], "trimmed": 0} + _put(session, view_id, lambda _prior: rec) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell the author their + # script was saved when it was not. + raise err(503, "store_unavailable", "the script view was NOT created") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.get("/script-views/{view_id}") +def get_script_view(view_id: str, session: Session = Depends(require_session)): + """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _database_ok(session, str(rec.get("database") or "")) + return {"view": _row(rec, source=True), "limits": _limits()} + + +@router.put("/script-views/{view_id}") +def update_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + def _mutate(prior): + prior = dict(prior or rec) + history = list(prior.get("history") or []) + history.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", "created": prior.get("updated") or ""}) + dropped = max(0, len(history) - MAX_HISTORY) + prior["history"] = history[dropped:] if dropped else history + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + if body.get("name"): + prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the new version was NOT saved") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.post("/script-views/{view_id}/revert") +def revert_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Put one view back to an earlier version. D-338 / W37-T46. + + ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole + point of the ticket rather than an implementation detail. The history is what lets a person + trust an agent with their code: if reverting destroyed the versions it stepped over, one + mistaken revert would cost exactly what the history existed to protect, and there would be no + way back from the way back. So this appends, and `restoredFrom` records where the text came + from — the same posture the agent-harness roll-back already takes, so a reader who has seen one + is not surprised by the other. + + ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history + entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old + version's text to re-PUT it. Handing the source out just so the client could send it straight + back would widen a payload for a round trip that does not need to exist, and would make the + revert non-atomic: two calls, and a failure between them leaves the person looking at code that + is not what is stored. + ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was + first saved, and a source that a later, stricter rule would now reject is exactly the source a + person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is + about what the code would DO and that must never be bypassed by a route. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + try: + want = int(body.get("version")) + except (TypeError, ValueError): + raise err(400, "no_version", "say which version to go back to") from None + + history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] + match = next((h for h in history if int(h.get("version") or 0) == want), None) + if match is None: + # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never + # written; the other was TRIMMED off the 40-deep history and the record even counts how + # many. Saying "that version is gone" for the first would be a lie a person could act on. + trimmed = int(rec.get("trimmed") or 0) + if trimmed and want <= trimmed: + raise err(410, "version_trimmed", + f"version {want} is older than this view's history keeps " + f"({trimmed} earlier versions have been dropped)") + raise err(404, "no_version", f"this view has no version {want}") + + source = str(match.get("source") or "") + refusal = sandbox.check_source(source) + if refusal is not None: + # An older version that today's sandbox refuses. The person is told which version and why, + # rather than being handed a 400 about code they did not just type. + raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") + + def _mutate(prior): + prior = dict(prior or rec) + history_now = list(prior.get("history") or []) + history_now.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", + "created": prior.get("updated") or ""}) + dropped = max(0, len(history_now) - MAX_HISTORY) + prior["history"] = history_now[dropped:] if dropped else history_now + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that + # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical + # code and no way to tell a revert from a coincidence. + prior["restoredFrom"] = want + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the roll-back was NOT saved") + return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} + + +@router.delete("/script-views/{view_id}") +def delete_script_view(view_id: str, session: Session = Depends(require_session)): + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + _put(session, view_id, lambda _prior: None) + return {"deleted": view_id} + + +@router.post("/script-views/{view_id}/run") +def run_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. + + ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser + executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key + before this function ever sees it, so a renderer cannot be talked into running something by a + script that was itself perfectly well behaved. + + ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess + wait on the event loop is a ten-second outage for the whole container. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + database = str(rec.get("database") or "") + # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. + # It is checked exactly as a stored one is, because "unsaved" is not a permission. + draft = (body or {}).get("source") if isinstance(body, dict) else None + source = _clean_source(draft) if draft else str(rec.get("source") or "") + # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and + # asking twice builds a registry topic's pool twice. The three refusals are translated below + # instead, which is the same wall reached through the same door. + + if not _RUN_SLOTS.acquire(blocking=False): + raise err(429, "busy", + f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " + f"Try again in a moment") + try: + out = sandbox.run_view(session.user, database, source, st=session.runtime) + finally: + _RUN_SLOTS.release() + + # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" + # answered 200 would be a permission decision the client has to go looking for, and every + # other door in this app answers 403 for it. Everything BELOW this line is a well-formed + # request whose ANSWER is that the script did not produce a view — that is a 200 with + # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. + if out.get("code") == "unknown_table": + raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") + if out.get("code") == "denied": + raise err(403, "forbidden", out.get("message") or "your account may not read that database") + if out.get("code") == "unresolvable": + refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") + refusal.detail["error"]["limit"] = out.get("limit") or {} + raise refusal + + answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), + "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), + "ms": int(out.get("ms") or 0), "code": out.get("code") or "", + # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits + # were applied; on a Windows host the memory and CPU ones were not, and a screen + # that claims an enforcement which did not happen is the failure the rule is about. + "caps": out.get("caps") or {}} + if not answer["ok"]: + answer["error"] = out.get("message") or "the script view did not produce a view" + return answer diff --git a/api/routes_tables.py b/api/routes_tables.py index 12779e6e22230f4221e06047381c33b8aa92b0c1..6239752892713dd6bbab4b44546847bb75841076 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -630,7 +630,23 @@ def ut_write_ctx(session: Session, table_key: str): return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, "hidden": hidden, "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⭐ W37-T12 — the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION + # list for `clean_measure_field`: without it a Metric create on this surface is + # refused `measure_not_offered` even though the read door offers the kind. + # ⛔ THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and + # materialises no rows at all (W30-T30 removed a full table build from the write + # path); computing measure values would put that work straight back. `measure_sets` + # stays empty for the same reason — a write is not a render. + # ⭐ AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN + # ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have + # different jobs: `measure_keys` (from `measures`) is the ADMISSION list + # `clean_filter_tree` uses — non-empty here, so a saved measure CONDITION survives the + # write, which is the failure that comment records for the customer path. Empty + # `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender", + # so the next `/workspace` — which DOES compute the sets, in `ut_assembly` — resolves + # it. Conservative in the right direction: one extra repaint, never a dropped filter. + "measures": ut_measures(table_key), + "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. "limits": limits, "defn": defn} @@ -741,16 +757,154 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", # so the closure covers this user's `custom_` and `measure_` columns too. fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src) + # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database + # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real + # lookback catalogue AND the cells to go with it; every other user table still serves `[]`, + # which is the honest answer for a hand-typed one rather than a descope. + today = time.strftime("%Y-%m-%d") + measures = ut_measures(table_key) + derived = aios_grid.cohort_cells(lists) # R9: this table's own lists + for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures, + session.runtime.measure_memo).items(): + derived.setdefault(_pid, {}).update(_cells) + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, "hidden": hidden, - # R9: the Cohorts column's cells from this table's own lists. - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + "derived": derived, + "measures": measures, + "measure_sets": _ut_measure_sets(table_key, views, measures, pids, today), + "today": today, # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test # for is a key a consumer forgets to test for, and this one carries a refusal. "limits": limits, "defn": defn} +def _ut_measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: # noqa: BLE001 + pass + + +def ut_measures(table_key): + """The lookback-measure OFFER for a user database, or `[]` — W37-T12 / owner item 4 (R1). + + ⭐ THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the + topic that DESCRIBES this database. A table with no entity topic, or a topic with no + `measures:` binding, gets `[]` — which is the honest answer for a hand-typed database and is + what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`). + + ⛔ NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model + (`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid, + so a list here would be a second definition of one fact — the argument `_rollup_source_offer` + makes about itself one screen down. + """ + try: + from harness import semantic as sem + topic = sem.topic_for_grid(table_key) + return sem.entity_measures(topic) if topic else [] + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-offer", e) + return [] + + +def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): + """`{pid: {field key: value}}` for this database's Metric columns. + + ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need + one. `sales_lines.agent` is `rp.agent_id` — a `res.partner` id — and a `ut_odoo_agents` row's + `pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys + `_id` on the partner id). So the group key and the pid are the same integer. The product grid + keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap: + the right answer differs per database and must be read off the binding, never assumed. + """ + import aios_grid + + mfields = aios_grid.measure_fields_of(fields) + if not mfields or not offer: + return {} + from harness import semantic as sem + from harness import windows as _wn + + topic = sem.topic_for_grid(table_key) + by_key = {m["key"]: m for m in offer} + by_window = {} + for f in mfields: + spec = f.get("measure") or {} + if spec.get("key") not in by_key: + continue + rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None + if rng is None: + continue # an unresolvable window is unanswerable — never widen to all time + by_window.setdefault((rng[0], rng[1]), []).append(f) + + out = {} + for (dfrom, dto), group in by_window.items(): + keys = tuple(sorted({f["measure"]["key"] for f in group})) + mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) + if mkey not in memo: + try: + memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, + date_to=dto, team_id=team_id, offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-column", e) + transient = False + try: + from harness import datastore as _ds + transient = not _ds.ready() + except Exception: # noqa: BLE001 + transient = False + if transient: + continue # a warming store must not memoise as a permanent failure + memo[mkey] = None + vals = memo[mkey] + if vals is None: + continue + # C1's empty-window rule for EVERY row, not only the grouped ones. + filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids} + for f in group: + k, fkey = f["measure"]["key"], f["key"] + for pid, cell in filled.items(): + if k in cell: + out.setdefault(pid, {})[fkey] = cell[k] + if len(memo) > 100: + memo.clear() + return out + + +def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None): + """`{rule id: [pid, …]}` for the measure CONDITIONS in this database's saved views. + + ⛔ SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same + list, so a non-empty `measures` also opens the measure condition in the filter builder — and a + rule id missing from `measureSets` renders PENDING ("Calculating…") and matches nothing, + permanently. Offer and sets are one feature. + """ + if not offer: + return {} + from harness import semantic as sem + + topic = sem.topic_for_grid(table_key) + admitted = {m["key"] for m in offer} + by_id = {} + for v in views or []: + for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted): + rid = str(rule.get("id") or "") + if rid: + by_id[rid] = rule + if not by_id: + return {} + try: + sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id, + keys_by_id=list(pids), offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-sets", e) + return {} + return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit()) + for rid, ks in sets.items()} + + def ut_label(defn, key, meta=None): """THE name of a user table, resolved ONCE (wave 20, item 6a). @@ -857,11 +1011,38 @@ def _rollup_source_offer(): "description": m.get("description") or ""}) out = [] + indeterminate = False for tkey, t in sorted(topics.items()): dims = ((t.get("store") or {}).get("dims") or {}) measures = by_topic.get(tkey) or [] if not dims or not measures: continue + # ⛔⛔ W37-T13 / /validate-wave 2026-08-19 — THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR + # DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE + # ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one + # door nobody re-read. `semantic._one_binding_offer` — the MEASURE-CATALOG door — refuses + # the same bindings with a stated cause, so the wave shipped **two doors onto one + # vocabulary with only one of them guarded**, and the unguarded one is the picker. + # MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four + # dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised + # `CatalogException: Table with name stock_move does not exist!` — five of the fifty + # (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog + # error, or a column that looks configured and is blank forever + # ([[permitted-is-not-answerable]]). + # ⚠ REUSES `_source_ready` rather than re-deriving readiness: one question, one + # normalizer. `None` (the store could not be asked) refuses TOO — fail closed, exactly as + # the measure door does. + _tbl = (t.get("store") or {}).get("table") + if _tbl: + _rdy = sem._source_ready(_tbl) + if _rdy is not True: + # ⛔ AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment + # records the measured version of this: a cold container with no mirror yet + # refused every binding, the refusal was memoised, and the grid served no measures + # long after the mirror arrived. Here the same shape would freeze a truncated + # PICKER for the life of the process. + indeterminate = indeterminate or (_rdy is None) + continue out.append({ "key": tkey, "label": t.get("label") or tkey, @@ -883,7 +1064,10 @@ def _rollup_source_offer(): windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} for k in ut.ROLLUP_SOURCE_WINDOWS] offer = {"topics": out, "windows": windows} - _ROLLUP_CACHE["offer"] = offer + # ⚠ Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped + # because the store could not be asked, this offer is a snapshot of an unknown, not a fact. + if not indeterminate: + _ROLLUP_CACHE["offer"] = offer return offer diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 8fb2db4f0aa9d0331ead406ab80a14bf146011f7..88cb8a8a615a0632d3002e8cdd709a7018b00539 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -24,6 +24,7 @@ Design notes: injected-HTML path remains a read/local-write fallback when component assets are absent. """ import json +import math import re from pathlib import Path @@ -89,6 +90,22 @@ def _round(v): return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v +def _round2(v): + """D-153 (W37, asked by lane B) — MONEY AND PERCENT KEEP THEIR DECIMALS. + + `_round`'s whole-number convention is right for counts and wrong for currency: `first_cost` + 3.47 shipped as `3`, so every money cell on every Odoo grid was a whole dollar. + + ⚠ `pct` IS AFFECTED TOO, and that is the half most likely to be dropped. `cells.ts` renders a + percent as `num(v).toFixed(1) + "%"`, so a `yoy_pct` of 12.34 was `round()`ed to 12 and painted + "12.0%" — one decimal of a number that HAS one, which reads as precision rather than as loss. + ⛔ NOT written as a `_round(v, nd=0)` default argument, deliberately: `round(v, 0)` returns a + FLOAT where `round(v)` returns an INT, so every integer column's wire shape would change under + an edit that looks purely cosmetic. + """ + return round(v, 2) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + # Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors # customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. # select — a single-select with its own `options` (the "Status" a user wants to add; distinct @@ -415,6 +432,52 @@ def _clean_formula(raw, valid_keys=None): return s +def _clean_geocode(raw): + """One `geocode` bag → the stored shape, or None (dropped). + + ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7). THE ONE validator for the geocode + pseudo-kind, called by BOTH field doors: `_field_extras` here (the Odoo grids) and + `core/user_tables._clean_field` via `_ag_geocode` (every `ut_*` user database). + + ⛔ IT WAS INLINE IN `_field_extras` UNTIL 2026-08-19 AND THAT IS WHY IT SHIPPED HALF-DONE. + `user_tables._clean_field` is a strict allowlist that returns only the bags it NAMES, so a + geocode column created on a user database was stored WITHOUT its `{addressField}` — the + *"created, named, configured, gone"* failure the `image` type shipped as in wave 19. Extracting + it rather than copying the rules is [[one-question-two-normalizers]]: a second copy is how the + two doors come to disagree about what a valid bag is. + + ⛔ WHOLE-KEY DROP (the `swipe` construction): a geocode column with no address column is not + a degraded column, it is a column no run can ever fill, and the honest state for that is the + unconfigured one the client refuses at `canCreate`. + + ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND + in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, + `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three + parity gates chain over those sets and none permits a partial landing. This is also why the + bag takes `format`'s posture and NOT `formula`'s: there is no type to pair it with, so an + absent bag is an ordinary text column rather than a refusal. + + ⚠ `addressField` is NOT checked against the table's keys: neither door is given `valid_keys`, + and the split this module already draws puts that loss on the surface that RENDERS. The + enricher must SAY the address column is gone rather than geocode an empty string. + """ + if not isinstance(raw, dict): + return None + addr = raw.get("addressField") + if not isinstance(addr, str) or not addr.strip(): + return None + out = {"addressField": addr.strip()[:80]} + # D-3 — the optional ISO 3166-1 alpha-2 hint, mapped straight onto the geocoder's + # `countrycodes` parameter. NORMALISED to upper case and stored only when it is exactly two + # letters: absent means "anywhere", and `""` would be that state's second spelling. A + # three-letter or numeric code is REFUSED rather than truncated, because a truncated code is + # a valid code for a different country. + cc = raw.get("country") + if isinstance(cc, str) and re.fullmatch(r"[A-Za-z]{2}", cc.strip() or ""): + out["country"] = cc.strip().upper() + return out + + def _field_extras(saved, ftype): """createdBy / permissions / format / scope — the validated passthrough the created strata share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it @@ -434,6 +497,27 @@ def _field_extras(saved, ftype): out["format"] = fmt if saved.get("scope") == "cohort": out["scope"] = "cohort" + # ── ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7): THE GEOCODE COLUMN'S CONFIG ── + # + # `{addressField}` — which column this one reads. WHOLE-KEY drop (the `swipe` construction): + # a geocode column with no address column is not a degraded one, it is a column no run can + # ever fill, and the honest state for that is the unconfigured one the client refuses at + # `canCreate`. + # + # ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND + # in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, + # `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three + # parity gates chain over those sets and none permits a partial landing. + # ⚠ `addressField` is NOT checked against this table's keys: `_field_extras` is not given + # `valid_keys`, and the split this module already draws puts that loss on the surface that + # RENDERS ("what each mode MEANS ... is the client's business"). The enricher must SAY the + # address column is gone rather than geocode an empty string. + # ⚠ EXTRACTED to `_clean_geocode` 2026-08-19 (/validate-wave, W37-T22 clause 1). The rules + # did not change; what changed is that `core/user_tables.py` now reaches THIS function through + # `_ag_geocode` instead of carrying a second copy. See `_clean_geocode`'s own note. + geo = _clean_geocode(saved.get("geocode")) + if geo: + out["geocode"] = geo corrected_from = saved.get("labelCorrectedFrom") correction_id = saved.get("labelCorrectionId") if (isinstance(corrected_from, str) and corrected_from.strip() @@ -752,7 +836,10 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): if field.get("derived"): continue # not on the pool row — filled from `derived` below v = r.get(k) - row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) + # D-153 — three roundings, not two: text/status/date pass through verbatim, currency + # and pct keep two decimals (`_round2`), everything else stays a whole number. + row[k] = v if field["type"] in {"text", "status", "date"} else ( + _round2(v) if field["type"] in {"currency", "pct"} else _round(v)) saved = overlays.get(str(pid), {}) or {} for field in overlay_fields: row[field["key"]] = saved.get(field["key"], "") @@ -789,6 +876,19 @@ COHORT_FIELD = '__cohort__' #: a set op reaching a column leaf would fall through the client engine's switch to "no #: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} + +#: ⭐⭐ WAVE-37 T26 (owner item 14, contract C4) — THE VIEW-MEMBERSHIP LEAF's reserved +#: pseudo-column and its operators. Mirrors `customer-grid/types.ts` VIEW_FIELD / VIEW_OPS. +#: +#: ⛔ DISJOINT FROM `FILTER_OPS`, for the reason COHORT_OPS is: `matchFilter` answers an operator +#: it does not know by NOT NARROWING, so an op that reached a column leaf would widen the result +#: under an authoritative count. Keeping the two vocabularies apart means a column leaf carrying +#: `inView` is dropped by the same fail-closed path that drops `dropTable`. +#: ⚠ The leaf names exactly ONE view (singular where a cohort leaf names a set), so there is no +#: all-or-nothing rule to get wrong here — the id either resolves or the leaf goes. +VIEW_FIELD = '__view__' +VIEW_OPS = {'inView', 'notInView'} +MAX_VIEW_ID = 64 #: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: #: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the #: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. @@ -912,8 +1012,32 @@ def _clean_rhs(raw, valid_keys): #: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by #: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered #: one is red too, so the hold cannot outlive its reason the way wave 27's did). +#: ⭐⭐ WAVE-37 T20 (owner item 10, ruling R2, contract C2) — 'script', and the reason it lands +#: here is NOT the one the PRD gives. The PRD says the Custom View is invisible because this set +#: omits the name so `_clean_display` drops the mode on every write. MEASURED, that write never +#: happens: a script view is a rail PROJECTION built client-side by +#: `CustomerGrid.tsx::scriptProjectionView`, its panel is gated on `activeScriptId !== null` +#: (a row in E's per-database store), there is no `displayMode === "script"` branch at all, and the +#: autosave effect reads `views.find(...)` while the projections are appended to `railViews` ALONE. +#: Nothing was ever dropped, because nothing was ever sent. +#: ⭐ WHAT THE NAME ACTUALLY BUYS is the PRECONDITION for the create door the owner asked for: a +#: script view can only be offered through the display-mode picker once a view STORED as +#: `mode: 'script'` survives a round trip, and until this line it could not. The picker entry +#: itself is W37-T41 (lane E) plus `CREATABLE_MODES`, and `_test/icons.test.ts::HELD_MODES` keeps +#: the hold with its release condition rewritten to say so. +#: ⚠ CONTRACT C2 MAKES THIS SET AND `customer-grid/types.ts::DISPLAY_MODES` IDENTICAL NAME FOR +#: NAME, and `verify_fields_contract.py` now asserts it in BOTH directions. That retires the staged +#: hold every mode from `timeseries` to `form` was landed with — a client-first name is now RED +#: rather than merely risky. The protection the hold gave is structural instead: both halves are in +#: one lane's fence, so they ship in one change, and whether a mode may be CREATED is a separate +#: question answered by `CREATABLE_MODES` / `HELD_MODES`. DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', - 'timeseries', 'catalog', 'swipe', 'form'} + 'timeseries', 'catalog', 'swipe', 'form', 'script'} + +#: W37-T20 (C2) — the cap on `display.script.id`. Script-view ids are minted server-side as +#: `'sv_' + secrets.token_urlsafe(9)` (`routes_script_views.py::create_script_view`), so ~15 chars; +#: 64 is a bound on a hostile payload, not an opinion about the format. +MAX_SCRIPT_ID = 64 #: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at #: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read @@ -926,6 +1050,166 @@ DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'char #: so one tenant setting its token to another tenant's value would silently receive that tenant's #: submissions. The token is minted server-side and lives in a bucket no client write can reach; #: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. +#: ⭐⭐ WAVE-37 T21 (owner items 1/12, rulings R5+R6, contract C3) — THE PER-VIEW CATALOG SPEC, +#: stored at `views[].config.display.catalog`. +#: +#: ⛔⛔ IT IS **`catalog`**, SINGULAR, AND IT IS NOT `catalogs`. Both keys live on `display`, they +#: are one letter apart, and that is the single most likely mistake anybody touching this file will +#: make. They are different things and BOTH are live: +#: · `catalogs` (wave-18 C6) — a LIST of up to 12 print artifacts, each with its own paper, +#: brand and pages of kind cover/intro/section/gallery. Authored CONTENT. UNTOUCHED by T21. +#: · `catalog` (wave-37 C3) — THIS view's catalog SETTINGS: which column is the picture, which +#: is the title (R5: per view, so two catalog views on one database may differ), plus the page +#: order and the free-placed items R6 asks for. +#: Neither replaces the other, and neither validator reads the other's key. +#: +#: ⛔ THE MIGRATION IS A RULE ABOUT WHAT THIS FUNCTION MUST **NOT** DO (R6, C3). An item authored +#: before free placement has no `x`/`y`, and an ABSENT coordinate must read back ABSENT — never +#: normalised to 0. Defaulting to zero would pile every pre-existing item at the origin, which is +#: "a catalog that silently loses its layout is a FAIL, not a migration" stated as code. The +#: RENDERER turns an absent coordinate into flow position N; the host's whole job is to not destroy +#: the distinction it reads. +#: ⚠ `w`/`h` carry the OPPOSITE asymmetry, deliberately: a zero WIDTH is an invisible item, so 0 +#: is refused and absent means "the renderer's default size". A zero X is the top-left corner and is +#: a real, storable position. Two keys of one shape with two rules — the `kanbanClamp` family — +#: which is why each is written out rather than looped. +MAX_CATALOG_SECTIONS = 20 +#: A bound on a hostile payload, not an opinion about the renderer's coordinate space: D owns what +#: the numbers MEAN. Stored to 2 decimals, finer than any print surface resolves. +MAX_CATALOG_COORD = 10000 +#: The page `order` bound. ABSENT => that page's ARRAY INDEX, and `order` WINS where present; ties +#: break by array index. Decided HERE rather than left to the renderer, because two readers guessing +#: differently about page order is exactly the divergence contract C3 exists to close. +MAX_CATALOG_ORDER = 9999 + + +def _clean_catalog_coord(raw, allow_zero): + """One free-placement number, or None. `allow_zero` False is the w/h leg (see the header). + + ⛔ THE ROUNDING IS `floor(v * 100 + 0.5) / 100` AND NOT `round(v, 2)`, AND THAT IS THE WHOLE + POINT OF THIS FUNCTION EXISTING SEPARATELY. Python's `round` is BANKER'S rounding and + JavaScript's `Math.round` is half-up-toward-+Infinity, so the two engines disagree on every + exact half: `round(0.125, 2)` is 0.12 here and 0.13 in the browser. One dragged item landing two + hundredths apart on the two engines is a `display` object that never compares equal — the + silent mirror drift contract C2 exists to prevent, arriving through arithmetic instead of + through a missing key. The expression below is `Math.floor(v * 100 + 0.5) / 100` written in + Python, evaluated in the same IEEE doubles, so the two agree bit for bit. + ⚠ AND AN INTEGRAL RESULT IS EMITTED AS AN `int`: `12.0` serialises as `12.0` in Python and + `12` in JSON.stringify, which is one stored value with two spellings. + """ + if isinstance(raw, bool) or not isinstance(raw, (int, float)): + return None + if raw != raw or raw in (float("inf"), float("-inf")): # NaN / +-inf + return None + val = math.floor(float(raw) * 100 + 0.5) / 100 + if val > MAX_CATALOG_COORD or val < -MAX_CATALOG_COORD: + return None + if not allow_zero and val <= 0: + return None + return int(val) if val == int(val) else val + + +def _clean_catalog_item(raw): + """One free-placed item: `{code, x?, y?, w?, h?}`. `code` REQUIRED; every coordinate optional. + + Key EMISSION ORDER is fixed and mirrored byte-for-byte by `types.ts::cleanCatalogSpec` — an + accepted save has to read back identically on both engines, and this pair is where that is + decided. + """ + if not isinstance(raw, dict): + return None + code = raw.get("code") + if not isinstance(code, str) or not code.strip(): + return None + out = {"code": code.strip()[:CATALOG_CODE_MAX]} + for key, allow_zero in (("x", True), ("y", True), ("w", False), ("h", False)): + val = _clean_catalog_coord(raw.get(key), allow_zero) + if val is not None: + out[key] = val + return out + + +def _clean_catalog_spec(raw, valid_keys): + """C3 — `display.catalog`, fail-closed. See the header block above for what it is NOT. + + Field refs follow the `dateField`/`stackField` rule this module already runs: a ref naming a + deleted column is dropped INDIVIDUALLY and the renderer falls back to its per-mode default. It + never costs the user their pages — the `charts` per-entry-drop precedent, applied to the one + payload here that is AUTHORED rather than derived from the rows. + """ + if not isinstance(raw, dict): + return None + out = {} + for ref in ("imageField", "titleField"): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + pages_raw = raw.get("pages") + pages = [] + if isinstance(pages_raw, list): + # ONE cumulative item budget across the whole spec, spent in page order then section order + # — the `_clean_catalogs` construction, so a 40-page catalog cannot smuggle 20,000 items + # past a per-page cap. + budget = MAX_CATALOG_CODES + seen_pages = set() + for raw_page in pages_raw: + if len(pages) >= MAX_CATALOG_PAGES: + break + if not isinstance(raw_page, dict): + continue + pid = str(raw_page.get("id") or "").strip()[:CATALOG_ID_MAX] + if not pid or pid in seen_pages: + continue + seen_pages.add(pid) + page = {"id": pid} + order = raw_page.get("order") + if (isinstance(order, int) and not isinstance(order, bool) + and 0 <= order <= MAX_CATALOG_ORDER): + page["order"] = order + sections_raw = raw_page.get("sections") + sections = [] + if isinstance(sections_raw, list): + seen_sections = set() + for raw_section in sections_raw: + if len(sections) >= MAX_CATALOG_SECTIONS: + break + if not isinstance(raw_section, dict): + continue + sid = str(raw_section.get("id") or "").strip()[:CATALOG_ID_MAX] + if not sid or sid in seen_sections: + continue + seen_sections.add(sid) + section = {"id": sid} + items_raw = raw_section.get("items") + items = [] + if isinstance(items_raw, list) and budget > 0: + for raw_item in items_raw: + item = _clean_catalog_item(raw_item) + if item is None: + continue + items.append(item) + budget -= 1 + if budget <= 0: + break + # ⚠ NOT de-duplicated by `code`. The same product legitimately appears twice in + # one section at two positions — that is what free placement IS — and wave-18's + # "deduped WITHIN a page" rule was written for a LISTING, where a repeat is a + # mistake. Here it is the feature. + if items: + section["items"] = items + sections.append(section) + # An EMPTY section is KEPT: a person adds a New Section and then drags into it, and + # dropping it would delete the thing they just made, between two autosaves. + if sections: + page["sections"] = sections + pages.append(page) + if pages: + out["pages"] = pages + # A spec that reduces to nothing is DROPPED, never stored as `{}`. "No image field, no title + # field, no pages" IS the unconfigured state, and `{}` would be its second spelling — the + # no-churn law every literal-only key in `_clean_display` follows. + return out or None + + FORM_ACCESS = ('public', 'emails') MAX_FORM_FIELDS = 60 MAX_FORM_EMAILS = 200 @@ -1373,7 +1657,11 @@ def _clean_display(raw, valid_keys): # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. mode = _LEGACY_MODES.get(mode, mode) out = {'mode': mode} - for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): + # ⭐ W37-T28 (owner item 15) — `coordField`: ONE column holding `"lat,lon"`, so a map is no + # longer a customers-only feature. It joins this loop rather than getting a clause of its own + # because it IS a field ref with the same drop-if-deleted rule; `customer-grid/types.ts` + # gains it in the SAME change, which is contract C2's whole point. + for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'): if raw.get(ref) in valid_keys: out[ref] = raw[ref] # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── @@ -1508,6 +1796,12 @@ def _clean_display(raw, valid_keys): catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) if catalogs: out['catalogs'] = catalogs + # ── ⭐⭐ WAVE-37 C3 (T21) — `catalog`, SINGULAR, and it is NOT the line above it. See the + # header on `_clean_catalog_spec` for the two-keys-one-letter-apart warning; this call site is + # where a reader is most likely to "fix" one into the other. + catalog = _clean_catalog_spec(raw.get('catalog'), valid_keys) + if catalog: + out['catalog'] = catalog # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. @@ -1536,6 +1830,26 @@ def _clean_display(raw, valid_keys): # `kanbanClamp` law) wearing a control that promises a choice. if left and right and left.casefold() != right.casefold(): out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} + # ── ⭐⭐ WAVE-37 T20 (owner item 10 / R2, contract C2): WHICH SCRIPT THIS VIEW RENDERS ── + # + # `{id}` — the row in E's per-database script store (`/api/v1/script-views`) whose source this + # view draws. WHOLE-KEY drop, the `swipe` construction rather than the `charts` one: a script + # view with no id is not a degraded script view, it is a panel with nothing to run, and the + # honest state for that is the unconfigured one the client already has a surface for. + # + # ⛔ THE ID IS DELIBERATELY NOT CHECKED AGAINST `valid_keys`, AND THAT IS NOT AN OVERSIGHT. + # `valid_keys` is this table's FIELD keys; a script id names a record in a DIFFERENT store that + # this function cannot reach. So the host bounds the shape and nothing else, and the renderer + # owns the one loss this is blind to — the script row was deleted — which it must SHOW rather + # than fall back to another script (the `viewModes.tsx` house rule `_clean_swipe`'s note cites). + # ⚠ Mirrored key-for-key by `customer-grid/types.ts::cleanDisplay`. That file rebuilds the + # config key by key on every autosave, so a key accepted HERE and unknown THERE is erased on the + # next column resize — which is the half of the round trip this docstring's W33 note warns about. + script = raw.get('script') + if isinstance(script, dict): + s_id = script.get('id') + if isinstance(s_id, str) and s_id.strip(): + out['script'] = {'id': s_id.strip()[:MAX_SCRIPT_ID]} # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. form = _clean_form(raw.get('form'), valid_keys) if form: @@ -1743,7 +2057,59 @@ def clean_item_folders(raw, folders, valid_ids): return out -def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): +def view_refs_of(nodes): + """C4 — every view id a tree REFERS to, deduped, in order. Mirrors `types.ts::viewRefsOf`. + + ⚠ An INACTIVE leaf (no view chosen yet) is not a reference, on both engines: counting one + would refuse a save the moment somebody opened the condition and had not picked anything. + """ + out = [] + for node in nodes or []: + if not isinstance(node, dict): + continue + if 'children' in node: + for vid in view_refs_of(node.get('children')): + if vid not in out: + out.append(vid) + continue + if node.get('colId') != VIEW_FIELD: + continue + vid = str(node.get('value') or '').strip()[:MAX_VIEW_ID] + if vid and vid not in out: + out.append(vid) + return out + + +def view_filter_cycle(view_id, nodes, filters_of): + """⭐⭐ W37-T26 / C4 — THE CYCLE REFUSAL's host half. Mirrors `types.ts::viewFilterCycle`. + + View X filtered on "is in View Y" where Y is filtered on X cannot be resolved by anything: + X needs Y needs X. Returns the CYCLE PATH (ids, starting and ending at `view_id`) or None. + + ⛔ THE CLIENT REFUSES THE SAME SHAPE AND THAT IS NOT DUPLICATION. The browser's guard is what + a PERSON sees and is the only one that can explain itself; this one is what makes the rule true + for a caller that POSTs a view directly, which is every guard-in-the-browser's blind spot. + ⚠ A view referring to ITSELF is a cycle of length one and is caught by the same walk — also + the case a person reaches most easily, by duplicating a view. + """ + def walk(current, path, seen): + refs = view_refs_of(nodes if current == view_id else filters_of(current)) + for nxt in refs: + if nxt == view_id: + return path + [nxt] + if nxt in seen: + continue + seen.add(nxt) + found = walk(nxt, path + [nxt], seen) + if found: + return found + return None + + return walk(view_id, [view_id], {view_id}) + + +def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, + visible_view_ids=None): """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). Returns a clean tree of leaf conditions ({colId, op, value, value2}) and @@ -1762,6 +2128,14 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): condition that can only match nothing, so `List is not [deleted]` would show an empty table forever with no way to tell why. `None` means this host has no cohorts, and then every cohort leaf is dropped: fail-closed, like every other unknown key. + + ⭐⭐ W37-T26 (C4) — `visible_view_ids` is the same parameter one leaf over: the views this + caller may name in a `__view__` membership leaf. A leaf naming anything else is DROPPED here + for the identical reason, and `None` (this host has no views to offer) drops every one. + ⚠ DEFAULTS TO `None` SO EVERY EXISTING CALLER IS BYTE-IDENTICAL. Three callers pass nothing + (`routes_admin`, `routes_query` twice) and each of them is a surface with no saved-view + vocabulary of its own; under the fail-closed rule they drop the leaf, which is correct rather + than merely safe. """ if budget is None: budget = [MAX_FILTER_NODES] @@ -1776,7 +2150,7 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): continue # too deep -> drop budget[0] -= 1 children = clean_filter_tree(node['children'], valid_keys, - depth + 1, budget, cohort_ids) + depth + 1, budget, cohort_ids, visible_view_ids) if children: out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', 'children': children}) @@ -1793,6 +2167,21 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): out.append({'colId': COHORT_FIELD, 'op': op, 'value': ','.join(named), 'value2': ''}) continue + if node.get('colId') == VIEW_FIELD: # C4: a view-membership leaf + # ⛔ THE SAME ALL-OR-NOTHING THE COHORT ARM ABOVE APPLIES, AND FOR THE SAME REASON. + # A leaf naming a view this caller cannot see is DROPPED rather than kept, because a + # kept one can only ever match nothing: `View is not [a view you cannot see]` would + # show an empty table forever with no way to tell why. Dropping it puts the view back + # in its honest unfiltered state, which is what `clean_filter_tree`'s own docstring + # already promises for cohorts. + # ⚠ `visible_views` is what the caller passes; `None` means this host does not know, + # and then EVERY view leaf is dropped — fail-closed, like every other unknown key. + v_op = node.get('op') + v_id = str(node.get('value') or '').strip()[:MAX_VIEW_ID] + if v_op in VIEW_OPS and v_id and v_id in (visible_view_ids or ()): + budget[0] -= 1 + out.append({'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''}) + continue if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: budget[0] -= 1 # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index ab13e52200c16ed629d388103bf5eb197a27e67f..ab4f4327eddb54820a861ecfdbdbf56dab750eb8 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -1,534 +1,548 @@ -{ - "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", - "fields": [ - { - "key": "customer", - "label": "Customer", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." - }, - { - "key": "partner_id", - "label": "Odoo ID", - "type": "int", - "source": "odoo", - "derived": true, - "default": false, - "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." - }, - { - "key": "odoo_status", - "label": "Odoo record", - "type": "status", - "source": "odoo", - "default": false, - "options": [ - "Active", - "Archived" - ], - "description": "Whether this customer still exists in Odoo. Archived means deleted there." - }, - { - "key": "agent", - "label": "Agent", - "type": "text", - "source": "odoo", - "default": true, - "description": "The sales agent who owns this account." - }, - { - "key": "dba", - "label": "DBA", - "type": "select", - "source": "odoo", - "default": false, - "options": [ - "Fisch", - "Royal", - "Both" - ], - "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." - }, - { - "key": "salesperson", - "label": "Salesperson", - "type": "text", - "source": "odoo", - "default": false, - "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." - }, - { - "key": "street", - "label": "Street", - "type": "text", - "source": "odoo", - "default": false, - "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." - }, - { - "key": "street2", - "label": "Street 2", - "type": "text", - "source": "odoo", - "default": false, - "description": "Second address line (suite, unit, floor) on the customer's Odoo address." - }, - { - "key": "city", - "label": "City", - "type": "text", - "source": "odoo", - "default": true, - "description": "City on the customer's Odoo address." - }, - { - "key": "state", - "label": "State", - "type": "text", - "source": "odoo", - "default": true, - "description": "State or province on the customer's Odoo address." - }, - { - "key": "country", - "label": "Country", - "type": "text", - "source": "odoo", - "default": false, - "description": "Country on the customer's Odoo address." - }, - { - "key": "zip", - "label": "ZIP", - "type": "text", - "source": "odoo", - "default": false, - "description": "Postal code on the customer's Odoo address." - }, - { - "key": "customer_since", - "label": "Customer since", - "type": "date", - "source": "odoo", - "default": false, - "description": "When this customer was first set up in Odoo." - }, - { - "key": "tags", - "label": "Tags", - "type": "text", - "source": "odoo", - "default": false, - "description": "Odoo labels on this customer, comma-separated." - }, - { - "key": "pricelist", - "label": "Price list", - "type": "text", - "source": "odoo", - "default": false, - "description": "The price list this customer buys on." - }, - { - "key": "payment_terms", - "label": "Payment terms", - "type": "text", - "source": "odoo", - "default": false, - "description": "Payment terms on this customer's account — Net 30, for example." - }, - { - "key": "last_order", - "label": "Last order", - "type": "date", - "source": "odoo", - "default": true, - "description": "Date of the most recent confirmed order." - }, - { - "key": "overdue_days", - "label": "Overdue days", - "type": "int", - "source": "odoo", - "default": true, - "description": "How many days late this customer is running against their own usual ordering rhythm." - }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, - { - "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", - "key": "ar_open", - "label": "AR current $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." - }, - { - "key": "ar_overdue", - "label": "AR overdue $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money past due — same basis as the Collections page." - }, - { - "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", - "key": "ar_outstanding", - "label": "AR outstanding $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." - }, - { - "key": "ar_exposure", - "label": "Credit exposure $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." - }, - { - "key": "ar_aged_1_30", - "label": "1-30 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_31_60", - "label": "31-60 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_61_90", - "label": "61-90 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_90_plus", - "label": "90+ days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "days_to_pay", - "label": "Days to pay", - "type": "int", - "source": "odoo", - "default": false, - "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." - }, - { - "key": "top_category", - "label": "Top category", - "type": "text", - "source": "odoo", - "default": false, - "description": "The category this customer spent the most on in the last 12 months." - }, - { - "key": "top_category_pct", - "label": "Top category %", - "type": "pct", - "source": "odoo", - "default": false, - "description": "Share of last-12-months spend that went to the top category." - }, - { - "key": "sku_count", - "label": "SKUs bought", - "type": "int", - "source": "odoo", - "default": false, - "description": "Distinct products bought in the last 12 months." - }, - { - "key": "top_sku", - "label": "Top SKU", - "type": "text", - "source": "odoo", - "default": false, - "description": "The product this customer spent the most on in the last 12 months." - }, - { - "key": "days_since", - "label": "Days since order", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days since the last confirmed order." - }, - { - "key": "typical_gap_days", - "label": "Typical gap days", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days this customer usually goes between orders, from their own history." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "description": "Your notes on this customer. Saved in this app only, visible only to you." - } - ], - "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.", - "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).", - "product_data": { - "identity": "pid", - "business_key": "code", - "fields": [ - { - "key": "code", - "label": "SKU", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." - }, - { - "key": "product", - "label": "Product", - "type": "text", - "source": "odoo", - "default": true, - "description": "Product name as it appears in Odoo." - }, - { - "key": "category", - "label": "Category", - "type": "select", - "source": "odoo", - "default": true, - "description": "Product category; '(uncategorized)' when Odoo carries none." - }, - { - "key": "supplier", - "label": "Supplier", - "type": "text", - "source": "overlay", - "default": true, - "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.", - "shared": true - }, - { - "key": "origin_country", - "label": "Country", - "type": "text", - "source": "overlay", - "default": false, - "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.", - "shared": true - }, - { - "key": "lead_days", - "label": "Lead time (days)", - "type": "int", - "source": "overlay", - "default": true, - "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.", - "shared": true - }, - { - "key": "first_cost", - "label": "First cost", - "type": "currency", - "source": "overlay", - "default": false, - "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.", - "shared": true - }, - { - "key": "price_fisch", - "label": "Fisch price", - "type": "currency", - "source": "odoo", - "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "price_royal_1", - "label": "Royal 1 price", - "type": "currency", - "source": "odoo", - "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "price_royal_2", - "label": "Royal 2 price", - "type": "currency", - "source": "odoo", - "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "rev_ytd", - "label": "Revenue YTD", - "type": "currency", - "source": "odoo", - "default": true, - "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is." - }, - { - "key": "rev_ly", - "label": "Revenue LY", - "type": "currency", - "source": "odoo", - "description": "Same period last year — seasonal wholesale compares like for like." - }, - { - "key": "yoy_pct", - "label": "YoY %", - "type": "pct", - "source": "odoo", - "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." - }, - { - "key": "qty_ytd", - "label": "Units YTD", - "type": "int", - "source": "odoo", - "description": "Units sold year to date." - }, - { - "key": "orders_ytd", - "label": "Orders YTD", - "type": "int", - "source": "odoo", - "description": "Distinct orders containing this SKU, year to date." - }, - { - "key": "on_hand", - "label": "On hand", - "type": "int", - "source": "odoo", - "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." - }, - { - "key": "unit_cost", - "label": "Unit cost", - "type": "currency", - "source": "odoo", - "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "inv_value", - "label": "Stock value", - "type": "currency", - "source": "odoo", - "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "qty_ltm", - "label": "Units LTM", - "type": "int", - "source": "odoo", - "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "dos", - "label": "Days of supply", - "type": "int", - "source": "odoo", - "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "cover_gap_d", - "label": "Cover gap (days)", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands." - }, - { - "key": "stock_bucket", - "label": "Stock status", - "type": "select", - "source": "odoo", - "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "needs_pricing", - "label": "Needs pricing", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "march_pricelist", - "label": "March pricelist", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "price_changes", - "label": "Price changes", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "closeouts", - "label": "Closeouts", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "shared": true, - "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace." - } - ] - } -} +{ + "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "partner_id", + "label": "Odoo ID", + "type": "int", + "source": "odoo", + "derived": true, + "default": false, + "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "options": [ + "Active", + "Archived" + ], + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "dba", + "label": "DBA", + "type": "select", + "source": "odoo", + "default": false, + "options": [ + "Fisch", + "Royal", + "Both" + ], + "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "street", + "label": "Street", + "type": "text", + "source": "odoo", + "default": false, + "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." + }, + { + "key": "street2", + "label": "Street 2", + "type": "text", + "source": "odoo", + "default": false, + "description": "Second address line (suite, unit, floor) on the customer's Odoo address." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", + "key": "ar_open", + "label": "AR current $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_outstanding", + "label": "AR outstanding $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "ar_aged_1_30", + "label": "1-30 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_31_60", + "label": "31-60 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_61_90", + "label": "61-90 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_90_plus", + "label": "90+ days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.", + "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).", + "product_data": { + "identity": "pid", + "business_key": "code", + "fields": [ + { + "key": "code", + "label": "SKU", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." + }, + { + "key": "product", + "label": "Product", + "type": "text", + "source": "odoo", + "default": true, + "description": "Product name as it appears in Odoo." + }, + { + "key": "category", + "label": "Category", + "type": "select", + "source": "odoo", + "default": true, + "description": "Product category; '(uncategorized)' when Odoo carries none." + }, + { + "key": "supplier", + "label": "Supplier", + "type": "text", + "source": "overlay", + "default": true, + "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.", + "shared": true + }, + { + "key": "origin_country", + "label": "Country", + "type": "text", + "source": "overlay", + "default": false, + "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.", + "shared": true + }, + { + "key": "lead_days", + "label": "Lead time (days)", + "type": "int", + "source": "overlay", + "default": true, + "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.", + "shared": true + }, + { + "key": "first_cost", + "label": "First cost", + "type": "currency", + "source": "overlay", + "default": false, + "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.", + "shared": true + }, + { + "key": "price_fisch", + "label": "Fisch price", + "type": "currency", + "source": "odoo", + "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "price_royal_1", + "label": "Royal 1 price", + "type": "currency", + "source": "odoo", + "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "price_royal_2", + "label": "Royal 2 price", + "type": "currency", + "source": "odoo", + "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "tier_prices", + "label": "Tier prices", + "type": "json", + "source": "odoo", + "description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name." + }, + { + "key": "units", + "label": "Units", + "type": "json", + "source": "odoo", + "description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier." + }, + { + "key": "rev_ytd", + "label": "Revenue YTD", + "type": "currency", + "source": "odoo", + "default": true, + "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is." + }, + { + "key": "rev_ly", + "label": "Revenue LY", + "type": "currency", + "source": "odoo", + "description": "Same period last year — seasonal wholesale compares like for like." + }, + { + "key": "yoy_pct", + "label": "YoY %", + "type": "pct", + "source": "odoo", + "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." + }, + { + "key": "qty_ytd", + "label": "Units YTD", + "type": "int", + "source": "odoo", + "description": "Units sold year to date." + }, + { + "key": "orders_ytd", + "label": "Orders YTD", + "type": "int", + "source": "odoo", + "description": "Distinct orders containing this SKU, year to date." + }, + { + "key": "on_hand", + "label": "On hand", + "type": "int", + "source": "odoo", + "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." + }, + { + "key": "unit_cost", + "label": "Unit cost", + "type": "currency", + "source": "odoo", + "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "inv_value", + "label": "Stock value", + "type": "currency", + "source": "odoo", + "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "qty_ltm", + "label": "Units LTM", + "type": "int", + "source": "odoo", + "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "dos", + "label": "Days of supply", + "type": "int", + "source": "odoo", + "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "cover_gap_d", + "label": "Cover gap (days)", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands." + }, + { + "key": "stock_bucket", + "label": "Stock status", + "type": "select", + "source": "odoo", + "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "needs_pricing", + "label": "Needs pricing", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "march_pricelist", + "label": "March pricelist", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "price_changes", + "label": "Price changes", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "closeouts", + "label": "Closeouts", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "shared": true, + "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace." + } + ] + } +} diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index b4b78bdc3ea98f45b86fbf5032c1bfd708894d4e..164f6a9c8898839a38a4a4a89d16acb1c6a8fcdc 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -801,9 +801,39 @@ def handle_one(event, ctx): # else is dropped here. Leaving it in would persist a condition the engine can only # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table # forever with nothing on screen explaining why. + # ⭐⭐ W37-T26 / CONTRACT C4 — `visible_view_ids` is the same PERMISSION list one leaf over: + # a `__view__` membership leaf naming a view this caller cannot see is dropped here, for + # the identical reason the cohort line above gives. Without it the arm is fail-closed and + # every view leaf is dropped on save, so the condition would vanish on the next read — + # D-90's shape exactly. + # ⚠ `ctx.visible_views` is already populated (`routes_grid.py` passes it, `:714` reads it), + # so this needs no new plumbing; it is the value nobody had handed to the validator. + # ⚠ Edited by lane C, not this file's owner: `platform/core/grid_events.py` is in NO fence + # and lane C was the last session open. See `mailbox/C.md` NOTE C-24. + _view_ids = {str(v.get('id')) for v in (visible_views or ()) + if isinstance(v, dict) and v.get('id')} + # ⛔⛔ THE CYCLE REFUSAL'S HOST HALF (C4), and it is the ticket's `done-when` rather than a + # guard bolted on afterwards. View X filtered on "is in View Y" where Y is filtered on X + # cannot be resolved by anything: X needs Y needs X. The browser refuses it too, with a + # sentence a person reads; this one is what makes the rule true for a caller that POSTs a + # view directly, which is every guard-in-the-browser's blind spot. + # ⚠ REFUSED, NOT REPAIRED. Dropping the offending leaf would save a view the person did not + # write and leave the panel showing a condition the config does not carry. + _cycle = _ag.view_filter_cycle( + view_id, cfg.get('filters'), + lambda vid: next((v.get('config', {}).get('filters') + for v in (visible_views or ()) + if isinstance(v, dict) and str(v.get('id')) == vid), None)) + if _cycle: + _refuse(ctx, 'view_cycle', + 'that would make a loop: ' + ' refers to '.join(_cycle) + + '. A view cannot filter on a view that filters on it, because neither can ' + 'be worked out without the other.', view_id) + return False filters = _ag.clean_filter_tree(cfg.get('filters'), valid_keys | set(measure_keys), - cohort_ids=set(cohort_ids)) + cohort_ids=set(cohort_ids), + visible_view_ids=_view_ids) sorts = [] for rule in list(cfg.get('sorts') or [])[:20]: if (isinstance(rule, dict) and rule.get('colId') in valid_keys diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 761b621379afae1e02987b1050d4cee636abbcdb..d1c6b511e71e8e892809a092b1e3001679b74f27 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -732,6 +732,24 @@ def _ag_formula(raw): return _agf._clean_formula(raw) +def _ag_geocode(raw): + """`aios_grid._clean_geocode`, reached exactly the way `_ag_formula` reaches its validator. + + ⭐⭐ W37-T22 clause 1, landed by /validate-wave 2026-08-19. The geocode PSEUDO-KIND was + offered in `ColumnMenu` and cleaned in `aios_grid._field_extras`, but THIS module — the + allowlist every `ut_*` create and patch goes through — named no `geocode` arm, so the bag was + stripped on every user database and the kind had to be gated OFF there. That gap is what made + the ticket's *"the kind is offered on ANY database that has a text field"* clause false. + + ⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and + `_clean_field`), for the reason `_ag_formula` states: `core` must not import `aios_grid` at + module level, and a third and fourth copy of `import aios_grid as _agX` is how one of them + ends up calling a different validator. + """ + import aios_grid as _agg + return _agg._clean_geocode(raw) + + #: Display formats a field may declare. ⛔ DISPLAY ONLY — none of these touches the stored value, #: which stays the scalar the fold or the mapper wrote. That separation is the whole safety of the #: feature: a `thousands` setting can never make a number wrong, only easier to read. @@ -934,6 +952,16 @@ def clean_fields(raw): entry['aiEnrich'] = ae # C3: derived, never typed beside the config. Same call as the other door. entry['automation'] = _field_agent_binding(entry) + # ⭐⭐ W37-T22 (R4 / C7) — THE GEOCODE BAG RIDES THIS DOOR TOO, written in the SAME change + # as `_clean_field`'s arm rather than a wave later. That is the only difference between + # this entry and the six above it (`pinned`, `code`, `link`/`rollup`, `formula`, + # `aiEnrich`), every one of which was a bag this validator dropped until somebody found a + # column that rendered and computed nothing forever. + # ⚠ NO type/bag pairing: geocode is a pseudo-kind on a `text` column (see `_clean_geocode`). + if entry is not None: + geo = (_ag_geocode(f.get('geocode')) if f.get('geocode') is not None else None) + if geo and ftype == 'text': + entry['geocode'] = geo if entry is None: seen.discard(key) continue @@ -2255,6 +2283,24 @@ def _clean_field(raw, previous=None): # a `field_agent` bag that disagrees with the column. Every OTHER automation bag is untouched. if ftype == 'ai_enrich': out['automation'] = _field_agent_binding(out) + # ⭐⭐ W37-T22 (R4 / C7) — THE GEOCODE PSEUDO-KIND'S BAG, added 2026-08-19 by /validate-wave. + # Posture is `format`'s and NOT `formula`'s, and that is the whole design: geocode has no + # FieldType to pair with (the column's `type` is `text`), so an absent bag is an ordinary text + # column rather than a refusal. Enforcing a pairing here would refuse every text column in + # every user database. + # ⚠ `'geocode' in raw` rather than `raw.get('geocode')`, exactly like `format` and + # `description`: an explicit `{}`/None CLEARS the address column while OMITTING the key keeps + # what is stored. A PATCH that sends only a label must not silently unconfigure the column — + # that is the same rule `agg` and `optionColors` are written for. + # ⚠ Inheritance is guarded on `ftype == 'text'` for the reason `prev_type == ftype` guards + # `link`/`rollup`/`formula`: retyping a geocode column to a number must not resurrect an + # address binding through `prev` that nothing will ever read again. + geocode_raw = (raw.get('geocode') if 'geocode' in raw + else (prev.get('geocode') if ftype == 'text' else None)) + if geocode_raw is not None: + geo = _ag_geocode(geocode_raw) + if geo and ftype == 'text': + out['geocode'] = geo return out diff --git a/platform/core/users.py b/platform/core/users.py index 0d37dfb63aea46b27d87a65720d0ce05c30f5463..18dd228c7b7904b55ba00ed2ea018f2b2cc75409 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -1,525 +1,532 @@ -"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json). - -Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) — never stored or logged in plaintext. -A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create -users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable. - -Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives -allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch). -""" -import os -import hmac -import hashlib -import secrets - -import core.store as store - -BU_LABELS = {5: 'Fisch', 6: 'Royal'} -_ITER = 200_000 - -#: Wave 15 C-PERM — the explicit-resolution marker. Mirrors `core.perm_scope.PERMS_VERSION`; -#: kept as a literal here so `users` does not import the permission layer it is read by. -PERMS_VERSION = 1 - - -def _hash(pw, salt): - return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex() - - -def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None, - perms=None, tenant='royal-imports', platform_admin=False): - salt = secrets.token_hex(16) - rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role, - 'bus': bus, 'active': active, 'modules': modules, - 'agent': agent or None, 'email': email or None, - # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every - # pre-wave record — no migration. Login binds the session to THIS value; the posted - # tenant field can hint but never override it. - 'tenant': str(tenant or 'royal-imports').strip().lower()} - if platform_admin is True: - # Wave 19 (R3): the PLATFORM-operator flag — half of `core.platform_admin`'s double lock - # (the other half is `tenant == 'loopable'`). Written ONLY for True, so every record that - # is not deliberately promoted keeps its pre-wave shape and answers False by absence. - # There is no UI writer and there never should be: it is set by provisioning, on purpose. - rec['platform_admin'] = True - if perms is not None: - # Wave 15 C-PERM. A record written WITH perms is migrated by construction — the marker - # and the block are set together, here, so no writer can create one without the other. - # (`core.perm_scope` reads an unmarked record as legacy, so a block without its marker - # would be silently ignored; a marker without a block would deny everything.) - rec['perms'] = perms - rec['perms_v'] = PERMS_VERSION - return rec - - -def registry(): - return store.get('users') - - -def ensure_bootstrap(): - """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet). - Idempotent; no-op if the store is unavailable (the app then falls back to the master-password - path in verify()). - - Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users') - is definitively False. A transient read failure at startup used to return {} and make this - re-seed just {admin} over the real accounts — that is the bug that wiped users on restart.""" - if not store.available(): - return - if store.exists('users'): # present, or uncertain -> never clobber - return - try: - reg = store.get('users', fresh=True) - except Exception: - return - if reg: - return - master = os.environ.get('APP_PASSWORD', '') - if not master: - return - try: - store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')}) - except Exception: - pass - - -def _public(username, u): - return {'username': username, 'name': u.get('name', username), - 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'), - 'modules': u.get('modules', 'all'), - 'agent': u.get('agent'), 'email': u.get('email'), - # Wave 18 (C1-TENANT): the session's tenant binding travels on the projection or it - # does not travel — the same rule the perms block states below. - 'tenant': str(u.get('tenant') or 'royal-imports').strip().lower(), - # Wave 14 C-AVATAR: the profile photo is public-safe by definition (it is served - # to every grid session via the workspace map); without it here the API session's - # user record silently drops it and /me can never show your own photo. - 'avatar': u.get('avatar') or None, - # ⛔ WAVE 15 C-PERM — THE WALL TRAVELS ON THIS PROJECTION OR IT DOES NOT TRAVEL. - # `deps._user_for` builds every API session from `_public()`, so a `perms` block - # dropped here is a restricted account served as an unrestricted one — silently, on - # every route, with nothing to notice. `perms_v` must ride ALONG WITH it and for the - # same reason inverted: the marker without the block denies everything, the block - # without the marker is ignored. Two keys, one fact, never separated. - # `verify_api` asserts a restricted user's SESSION OBJECT carries both, at the mount - # rather than by grep — a projection is exactly the kind of wiring that looks - # present in three files and is absent in the one that runs. - **({'perms': u['perms']} if isinstance(u.get('perms'), dict) else {}), - **({'perms_v': int(u['perms_v'] or 0)} if u.get('perms_v') else {}), - # ⛔ WAVE 19 R3 — THE SAME RULE, ON A NEW FIELD. `deps._user_for` builds every API - # session from this projection, so the platform-admin flag travels here or - # `core.platform_admin.is_platform_admin(session.user)` is blind and the Loopable - # admin plane 403s its own operator. Carried ONLY when the record says True, so a - # session dict for any other account is byte-identical to its pre-wave shape. - # Not a client leak: `routes_auth._public_user` is a whitelist projection and does - # not name this key, so it reaches no browser via /login or /me — the client's copy - # is the separate `platformAdmin` bool on GET /settings, which is derived from this. - **({'platform_admin': True} if u.get('platform_admin') is True else {}), - 'epoch': int(u.get('epoch') or 0)} - - -# ------------------------------------------------------------------ session revocation (X3) -# The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to -# delete from, so "log this user out everywhere" needs a number that lives with the account. The -# cookie carries the epoch it was minted under; bumping the account's epoch makes every -# outstanding cookie for that user fail verification on its next use. Absent == 0, so every -# record written before this wave is valid without a migration. -def epoch(username): - """The current session epoch for `username`. None when there is no such account. - - None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" — - which the session verifier must treat as a reason to refuse, not as a default to compare - against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles - that case explicitly rather than inventing an epoch for it here.) - """ - username = (username or '').strip().lower() - try: - u = (store.get('users') or {}).get(username) - except Exception: - return None - return int((u or {}).get('epoch') or 0) if u else None - - -def bump_epoch(username): - """Revoke every outstanding API session for this account.""" - username = (username or '').strip().lower() - - def _set(reg): - u = reg.get(username) - if u: - u['epoch'] = int(u.get('epoch') or 0) + 1 - return reg - store.update('users', _set) - - -def verify(username, pw): - """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the - 'admin' login even if the store is unreachable, so the owner is never locked out.""" - username = (username or '').strip().lower() - if not username or not pw: - return None - master = os.environ.get('APP_PASSWORD', '') - try: - # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once - reg = store.get('users', fresh=True) - except Exception: - reg = {} - u = reg.get(username) - if u is None and '@' in username: - # Wave 18 (R1): the login box takes a username OR an email — admin@nurilab.id signs in - # without knowing the slug an admin chose. First case-insensitive email match wins; - # ambiguity is an admin data problem, not a login feature. - for k, r in reg.items(): - if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username: - username, u = k, r - break - if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']): - return _public(username, u) - # emergency master: admin + APP_PASSWORD always works (covers first run / store outage) - if username == 'admin' and master and hmac.compare_digest(str(pw), master): - # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie - # the API mints from this dict AGREES with the stored account. - # - # This is not what stops the emergency lockout — `deps._user_for`'s master fallback does - # that, and a negative control confirmed the lockout is gone with or without this line. - # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the - # record falls through to that master fallback, which hands back a SYNTHETIC identity - # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would - # therefore be silently WIDENED to consolidated, all-module access for the life of that - # session. Matching the epoch means the record branch wins and the account's real scope - # applies, leaving the master fallback as the true last resort it is meant to be. - # 0 when the store is unreachable, which is the case this branch was written for. - return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all', - 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)} - return None - - -def create_user(username, pw, name, role='user', bus='all', modules='all', - agent=None, email=None, tenant=None, platform_admin=None): - """Create — or, from app.py's dialog, OVERWRITE — an account. - - ⛔ X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh - record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset - the epoch to 0, and every cookie minted before that account's last password rotation started - verifying again — a silent un-revocation. `app.py`'s "Add / update a user" calls this function - for BOTH add and update, so the hole was reachable from the shipped UI. - - Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so - the carry-and-bump lives here rather than in each caller: an overwrite is at least as - session-invalidating as a password change, and it usually IS one. - """ - username = (username or '').strip().lower() - if not username or not pw: - raise ValueError('username and password are required') - - def _add(reg): - prior = reg.get(username) or {} - rec = _record(pw, name or username, role, bus, modules=modules, - agent=agent, email=email, - # An overwrite that names no tenant KEEPS the account's company — a - # rename must never quietly move a user between tenants. - tenant=(tenant or prior.get('tenant') or 'royal-imports'), - # Wave 19 (R3): CARRIED, for the same reason `epoch` is carried below — - # `_record` builds a FRESH record, so re-running the provisioner (it is - # documented as idempotent) or saving an account through this function - # would silently DEMOTE a platform admin and lock the operator out of - # their own plane. None = leave as it was; True/False = set it deliberately. - platform_admin=(prior.get('platform_admin') is True - if platform_admin is None else platform_admin is True)) - if prior: - rec['epoch'] = int(prior.get('epoch') or 0) + 1 - reg[username] = rec - return reg - store.update('users', _add) - - -def set_password(username, pw): - username = (username or '').strip().lower() - - def _set(reg): - u = reg.get(username) - if u: - u['salt'] = secrets.token_hex(16) - u['hash'] = _hash(pw, u['salt']) - # X3: a password change revokes every outstanding API session for the account. Bumped - # INSIDE the same read-modify-write as the hash so the two can never disagree — a - # separate update() could rotate the password and leave old cookies live if the second - # write failed. - u['epoch'] = int(u.get('epoch') or 0) + 1 - return reg - store.update('users', _set) - - -def set_active(username, active): - username = (username or '').strip().lower() - - def _set(reg): - if username in reg: - reg[username]['active'] = bool(active) - # X3: DEACTIVATION must kill live sessions, not just future logins — otherwise a - # disabled account keeps working until its cookie expires. Bumped on reactivation too: - # cheap, and it means a re-enabled account never resurrects a stale cookie. - reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1 - return reg - store.update('users', _set) - - -def set_platform_admin(username, on): - """Promote/demote a PLATFORM administrator (wave 19, R3) without touching the password. - - The narrow write, deliberately: `create_user` is the destructive path (fresh salt, fresh - hash, bumped epoch) and promoting somebody should not sign them out or rotate a credential. - Cleared by REMOVING the key, so a demoted record goes back to its pre-wave shape rather than - carrying a `False` that reads as "somebody considered this". - - ⚠ This is the flag only. It grants nothing on its own — `core.platform_admin` also demands - the `loopable` tenant, and there is no code path anywhere that moves an account between - tenants, which is what makes the second lock hold. - """ - username = (username or '').strip().lower() - - def _set(reg): - u = reg.get(username) - if u: - if on is True: - u['platform_admin'] = True - else: - u.pop('platform_admin', None) - return reg - store.update('users', _set) - - -# ------------------------------------------------------------------ activity stamps (wave 19 R4) -# "When did this account last sign in, and is anyone actually using it?" — the two questions the -# Loopable admin plane exists to answer and that NOTHING in the product could answer before this -# wave (there is no login history, no audit log, no request log anywhere). -# -# ⛔ THIS IS THE FIRST HIGH-FREQUENCY WRITER `users.json` HAS EVER HAD, and that bucket also holds -# every password hash, `active`, `epoch` and the permission blocks. Three rules follow, and the -# second one is a correctness rule, not a performance one: -# -# 1. **In-place mutation of ONE key.** Never `_record()`, never a whole-record replace: a stamp -# that rebuilt the record would reset the salt/hash (locking the user out) or the epoch -# (silently un-revoking every cookie ever minted for them). `set_password`'s docstring -# explains why that class of bug is worth naming out loud. -# -# 2. **SYNCHRONOUS FLUSH — deliberately NOT the async path, and this reverses my first draft.** -# `store.update(flush='async')` rebases on the PROCESS CACHE once a key is `_owned` -# (`core/store.py:284`) and its worker uploads that whole cached blob. For a -# table-workspace key, written by one process, that is exactly right. For `users` it is a -# silent-revert machine: tenant #0's Streamlit host writes the SAME file, so an API process -# holding a cache from an hour ago would, on its next stamp, upload a blob in which a -# password rotation or a deactivation performed in the other host simply never happened. -# A telemetry stamp must not be able to resurrect a disabled account. `flush='sync'` does a -# FRESH strict read inside the store's lock and then uploads, which is the same discipline -# every other `users` writer already uses. -# -# 3. **OFF THE REQUEST THREAD, so rule 2 costs nothing.** A sync commit is a hub round-trip, and -# neither a sign-in nor a random request an hour later should wait for it. Each stamp runs on -# a short-lived daemon thread; `flush_stamps()` is how a test or a shutdown waits for them. -# Everything is fail-silent: a stamp is telemetry and may never turn a good login into a -# failed one — the plane shows an honest "never" instead. -def _now_iso(): - import datetime as _dt - return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec='seconds') - - -#: Live stamp threads, so `flush_stamps()` can join them. Bounded by construction — one thread per -#: stamp, and a stamp is at most one per login plus one per account per hour per process. -_STAMPS = [] -_STAMPS_LOCK = __import__('threading').Lock() - - -def _stamp(username, fields): - """Merge `fields` into ONE account record, on a background thread, with a fresh read.""" - username = (username or '').strip().lower() - if not username or not fields: - return None - - def _set(reg): - u = reg.get(username) - if isinstance(u, dict): - u.update(fields) - return reg - - def _work(): - try: - store.update('users', _set) # sync: fresh strict read + blocking upload - except Exception: - pass # a lost stamp is a lost stamp, never an error - import threading - t = threading.Thread(target=_work, daemon=True, name=f'user-stamp:{username}') - with _STAMPS_LOCK: - _STAMPS[:] = [x for x in _STAMPS if x.is_alive()] - _STAMPS.append(t) - t.start() - return t - - -def flush_stamps(timeout=10.0): - """Block until outstanding stamp writes have been applied. For gates and shutdown hooks — - the app never needs it, exactly like `core.store.flush`.""" - with _STAMPS_LOCK: - pending = list(_STAMPS) - for t in pending: - t.join(timeout=timeout) - return all(not t.is_alive() for t in pending) - - -def touch_login(username, when=None): - """Stamp `last_login` (ISO-8601, UTC, OFFSET-BEARING) on a SUCCESSFUL login. - - `username` must be the RESOLVED account key, not what the person typed: `verify()` accepts an - email address and resolves it to the registry key, so stamping the typed identifier would - write a stamp onto a key that does not exist and create a phantom account in the registry. - - `last_active` rides along — signing in IS activity, and setting both here means the plane's - two columns agree the moment somebody logs in rather than an hour later. - """ - stamp = when or _now_iso() - return _stamp(username, {'last_login': stamp, 'last_active': stamp}) - - -def touch_active(username, when=None): - """Stamp `last_active` — "this session did something". Throttled BY THE CALLER (`deps.py` - holds a process-local last-seen map), so this is not a store round-trip per request.""" - return _stamp(username, {'last_active': when or _now_iso()}) - - -def set_access(username, role=None, bus=None, modules=None, agent=None, email=None, - name=None, perms=None): - """Update access fields. agent/email: pass '' to clear, None to leave unchanged — - the user↔agent link scopes the Customer List page / digests to that agent's book. - - `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all: - a display name could previously only be changed by re-creating the record through - `create_user`, i.e. by also resetting the password (and, before the fix above, the session - epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one.""" - username = (username or '').strip().lower() - - def _set(reg): - u = reg.get(username) - if u: - if name is not None: - u['name'] = name - if role is not None: - u['role'] = role - if bus is not None: - u['bus'] = bus - if modules is not None: - u['modules'] = modules - if agent is not None: - u['agent'] = agent or None - if email is not None: - u['email'] = email or None - if perms is not None: - # Wave 15 C-PERM. Writing perms MIGRATES the record: the marker goes on in the - # same read-modify-write, so a record can never end up with one and not the - # other (see `_record`). Whole-block replace, matching the PUT route's shape — - # a merge would make "remove this restriction" unexpressible. - u['perms'] = perms - u['perms_v'] = PERMS_VERSION - return reg - store.update('users', _set) - - -def allowed_bus_labels(user): - """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no - 'All', so the other BU is never reachable); multiple -> All + each.""" - bus = (user or {}).get('bus', 'all') - if bus == 'all': - return ['All', 'Fisch', 'Royal'] - labels = [BU_LABELS[b] for b in bus if b in BU_LABELS] - if not labels: - return ['All', 'Fisch', 'Royal'] - return (['All'] + labels) if len(labels) > 1 else labels - - -def assignable_people(tenant=None): - """Display names for `user`-typed overlay columns — the tenant's ACTIVE accounts. - - Moved from app.py (2026-07-31) so both hosts serve the same choices. Resolved on every - call rather than persisted with the column: a snapshot would keep offering people who - have left and never offer people who joined. Deactivated accounts are excluded; a value - already stored on a row is untouched — history should still say who owned something. - - Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry - is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a - cross-tenant name leak. - - ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and - return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was - written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at - EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform - roster one forgotten kwarg away, with no error to notice - ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing). - ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass - `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate - changes — and a future caller that forgets gets an empty picker somebody notices instead of a - leak nobody does. - """ - return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)}) - - -def set_avatar(username, data_url): - """Set (or clear, with None/'') the user's profile photo — a data URL (wave 14 C-AVATAR, - [[loopable-wave14-split]] item 11). Stored VERBATIM; the API route owns validation (mime + - decoded size) because this value is served back to every grid session. Cleared by removing - the key, so records without a photo keep their pre-wave shape.""" - username = (username or '').strip().lower() - - def _set(reg): - u = reg.get(username) - if u: - if data_url: - u['avatar'] = str(data_url) - else: - u.pop('avatar', None) - return reg - store.update('users', _set) - - -def avatar_map(tenant=None): - """{display name -> avatar data URL} for ACTIVE accounts with a photo — the companion of - `assignable_people()`, keyed by the SAME vocabulary: a `user` cell stores the display - name, so the display name is the only join a renderer has. Two active accounts sharing a - display name share one option; the first WITH a photo wins the key rather than a coin - flip deciding whether the option has a face. `tenant` scopes it exactly as - `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed - blank, which both take from the ONE resolver below rather than each writing `if want and …`. - """ - out = {} - for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]): - av = u.get('avatar') - nm = str(u.get('name') or username) - if av and nm not in out: - out[nm] = str(av) - return out - - -def _tenant_accounts(tenant): - """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED. - - ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice, - identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a - blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of - them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of - the same predicate and is lane C's to fold in. - - ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a - pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public` - normalises it the same way. What changed is the WANT side — a blank want is now a refusal - rather than a wildcard. - """ - want = str(tenant or '').strip().lower() - if not want: - # Not an error and not everything: nobody. See `assignable_people`'s note — the one - # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6. - return [] - try: - reg = registry() or {} - except Exception: - return [] - return [(n, u) for n, u in reg.items() - if isinstance(u, dict) and u.get('active') is not False - and str(u.get('tenant') or 'royal-imports').strip().lower() == want] +"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json). + +Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) — never stored or logged in plaintext. +A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create +users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable. + +Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives +allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch). +""" +import os +import hmac +import hashlib +import secrets + +import core.store as store + +BU_LABELS = {5: 'Fisch', 6: 'Royal'} +_ITER = 200_000 + +#: Wave 15 C-PERM — the explicit-resolution marker. Mirrors `core.perm_scope.PERMS_VERSION`; +#: kept as a literal here so `users` does not import the permission layer it is read by. +PERMS_VERSION = 1 + + +def _hash(pw, salt): + return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex() + + +def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None, + perms=None, tenant='royal-imports', platform_admin=False): + salt = secrets.token_hex(16) + rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role, + 'bus': bus, 'active': active, 'modules': modules, + 'agent': agent or None, 'email': email or None, + # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every + # pre-wave record — no migration. Login binds the session to THIS value; the posted + # tenant field can hint but never override it. + 'tenant': str(tenant or 'royal-imports').strip().lower()} + if platform_admin is True: + # Wave 19 (R3): the PLATFORM-operator flag — half of `core.platform_admin`'s double lock + # (the other half is `tenant == 'loopable'`). Written ONLY for True, so every record that + # is not deliberately promoted keeps its pre-wave shape and answers False by absence. + # There is no UI writer and there never should be: it is set by provisioning, on purpose. + rec['platform_admin'] = True + if perms is not None: + # Wave 15 C-PERM. A record written WITH perms is migrated by construction — the marker + # and the block are set together, here, so no writer can create one without the other. + # (`core.perm_scope` reads an unmarked record as legacy, so a block without its marker + # would be silently ignored; a marker without a block would deny everything.) + rec['perms'] = perms + rec['perms_v'] = PERMS_VERSION + return rec + + +def registry(): + return store.get('users') + + +def ensure_bootstrap(): + """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet). + Idempotent; no-op if the store is unavailable (the app then falls back to the master-password + path in verify()). + + Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users') + is definitively False. A transient read failure at startup used to return {} and make this + re-seed just {admin} over the real accounts — that is the bug that wiped users on restart.""" + if not store.available(): + return + if store.exists('users'): # present, or uncertain -> never clobber + return + try: + reg = store.get('users', fresh=True) + except Exception: + return + if reg: + return + master = os.environ.get('APP_PASSWORD', '') + if not master: + return + try: + store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')}) + except Exception: + pass + + +def _public(username, u): + return {'username': username, 'name': u.get('name', username), + 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'), + 'modules': u.get('modules', 'all'), + 'agent': u.get('agent'), 'email': u.get('email'), + # Wave 18 (C1-TENANT): the session's tenant binding travels on the projection or it + # does not travel — the same rule the perms block states below. + 'tenant': str(u.get('tenant') or 'royal-imports').strip().lower(), + # Wave 14 C-AVATAR: the profile photo is public-safe by definition (it is served + # to every grid session via the workspace map); without it here the API session's + # user record silently drops it and /me can never show your own photo. + 'avatar': u.get('avatar') or None, + # ⛔ WAVE 15 C-PERM — THE WALL TRAVELS ON THIS PROJECTION OR IT DOES NOT TRAVEL. + # `deps._user_for` builds every API session from `_public()`, so a `perms` block + # dropped here is a restricted account served as an unrestricted one — silently, on + # every route, with nothing to notice. `perms_v` must ride ALONG WITH it and for the + # same reason inverted: the marker without the block denies everything, the block + # without the marker is ignored. Two keys, one fact, never separated. + # `verify_api` asserts a restricted user's SESSION OBJECT carries both, at the mount + # rather than by grep — a projection is exactly the kind of wiring that looks + # present in three files and is absent in the one that runs. + **({'perms': u['perms']} if isinstance(u.get('perms'), dict) else {}), + **({'perms_v': int(u['perms_v'] or 0)} if u.get('perms_v') else {}), + # ⛔ WAVE 19 R3 — THE SAME RULE, ON A NEW FIELD. `deps._user_for` builds every API + # session from this projection, so the platform-admin flag travels here or + # `core.platform_admin.is_platform_admin(session.user)` is blind and the Loopable + # admin plane 403s its own operator. Carried ONLY when the record says True, so a + # session dict for any other account is byte-identical to its pre-wave shape. + # Not a client leak: `routes_auth._public_user` is a whitelist projection and does + # not name this key, so it reaches no browser via /login or /me — the client's copy + # is the separate `platformAdmin` bool on GET /settings, which is derived from this. + **({'platform_admin': True} if u.get('platform_admin') is True else {}), + 'epoch': int(u.get('epoch') or 0)} + + +# ------------------------------------------------------------------ session revocation (X3) +# The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to +# delete from, so "log this user out everywhere" needs a number that lives with the account. The +# cookie carries the epoch it was minted under; bumping the account's epoch makes every +# outstanding cookie for that user fail verification on its next use. Absent == 0, so every +# record written before this wave is valid without a migration. +def epoch(username): + """The current session epoch for `username`. None when there is no such account. + + None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" — + which the session verifier must treat as a reason to refuse, not as a default to compare + against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles + that case explicitly rather than inventing an epoch for it here.) + """ + username = (username or '').strip().lower() + try: + u = (store.get('users') or {}).get(username) + except Exception: + return None + return int((u or {}).get('epoch') or 0) if u else None + + +def bump_epoch(username): + """Revoke every outstanding API session for this account.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def verify(username, pw): + """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the + 'admin' login even if the store is unreachable, so the owner is never locked out.""" + username = (username or '').strip().lower() + if not username or not pw: + return None + master = os.environ.get('APP_PASSWORD', '') + try: + # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once + reg = store.get('users', fresh=True) + except Exception: + reg = {} + u = reg.get(username) + # ⛔ KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT. + # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a + # different account's `email` field at the same time. The key wins, here, by construction: this + # branch is only reached when `reg.get(username)` missed. `routes_admin.py::create_user` refuses + # to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the + # ambiguity cannot be introduced through the product; this line is what decides it for any + # record that arrived some other way. + if u is None and '@' in username: + # Wave 18 (R1): the login box takes a username OR an email — admin@nurilab.id signs in + # without knowing the slug an admin chose. First case-insensitive email match wins; + # ambiguity is an admin data problem, not a login feature. + for k, r in reg.items(): + if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username: + username, u = k, r + break + if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']): + return _public(username, u) + # emergency master: admin + APP_PASSWORD always works (covers first run / store outage) + if username == 'admin' and master and hmac.compare_digest(str(pw), master): + # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie + # the API mints from this dict AGREES with the stored account. + # + # This is not what stops the emergency lockout — `deps._user_for`'s master fallback does + # that, and a negative control confirmed the lockout is gone with or without this line. + # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the + # record falls through to that master fallback, which hands back a SYNTHETIC identity + # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would + # therefore be silently WIDENED to consolidated, all-module access for the life of that + # session. Matching the epoch means the record branch wins and the account's real scope + # applies, leaving the master fallback as the true last resort it is meant to be. + # 0 when the store is unreachable, which is the case this branch was written for. + return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all', + 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)} + return None + + +def create_user(username, pw, name, role='user', bus='all', modules='all', + agent=None, email=None, tenant=None, platform_admin=None): + """Create — or, from app.py's dialog, OVERWRITE — an account. + + ⛔ X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh + record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset + the epoch to 0, and every cookie minted before that account's last password rotation started + verifying again — a silent un-revocation. `app.py`'s "Add / update a user" calls this function + for BOTH add and update, so the hole was reachable from the shipped UI. + + Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so + the carry-and-bump lives here rather than in each caller: an overwrite is at least as + session-invalidating as a password change, and it usually IS one. + """ + username = (username or '').strip().lower() + if not username or not pw: + raise ValueError('username and password are required') + + def _add(reg): + prior = reg.get(username) or {} + rec = _record(pw, name or username, role, bus, modules=modules, + agent=agent, email=email, + # An overwrite that names no tenant KEEPS the account's company — a + # rename must never quietly move a user between tenants. + tenant=(tenant or prior.get('tenant') or 'royal-imports'), + # Wave 19 (R3): CARRIED, for the same reason `epoch` is carried below — + # `_record` builds a FRESH record, so re-running the provisioner (it is + # documented as idempotent) or saving an account through this function + # would silently DEMOTE a platform admin and lock the operator out of + # their own plane. None = leave as it was; True/False = set it deliberately. + platform_admin=(prior.get('platform_admin') is True + if platform_admin is None else platform_admin is True)) + if prior: + rec['epoch'] = int(prior.get('epoch') or 0) + 1 + reg[username] = rec + return reg + store.update('users', _add) + + +def set_password(username, pw): + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + u['salt'] = secrets.token_hex(16) + u['hash'] = _hash(pw, u['salt']) + # X3: a password change revokes every outstanding API session for the account. Bumped + # INSIDE the same read-modify-write as the hash so the two can never disagree — a + # separate update() could rotate the password and leave old cookies live if the second + # write failed. + u['epoch'] = int(u.get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def set_active(username, active): + username = (username or '').strip().lower() + + def _set(reg): + if username in reg: + reg[username]['active'] = bool(active) + # X3: DEACTIVATION must kill live sessions, not just future logins — otherwise a + # disabled account keeps working until its cookie expires. Bumped on reactivation too: + # cheap, and it means a re-enabled account never resurrects a stale cookie. + reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1 + return reg + store.update('users', _set) + + +def set_platform_admin(username, on): + """Promote/demote a PLATFORM administrator (wave 19, R3) without touching the password. + + The narrow write, deliberately: `create_user` is the destructive path (fresh salt, fresh + hash, bumped epoch) and promoting somebody should not sign them out or rotate a credential. + Cleared by REMOVING the key, so a demoted record goes back to its pre-wave shape rather than + carrying a `False` that reads as "somebody considered this". + + ⚠ This is the flag only. It grants nothing on its own — `core.platform_admin` also demands + the `loopable` tenant, and there is no code path anywhere that moves an account between + tenants, which is what makes the second lock hold. + """ + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + if on is True: + u['platform_admin'] = True + else: + u.pop('platform_admin', None) + return reg + store.update('users', _set) + + +# ------------------------------------------------------------------ activity stamps (wave 19 R4) +# "When did this account last sign in, and is anyone actually using it?" — the two questions the +# Loopable admin plane exists to answer and that NOTHING in the product could answer before this +# wave (there is no login history, no audit log, no request log anywhere). +# +# ⛔ THIS IS THE FIRST HIGH-FREQUENCY WRITER `users.json` HAS EVER HAD, and that bucket also holds +# every password hash, `active`, `epoch` and the permission blocks. Three rules follow, and the +# second one is a correctness rule, not a performance one: +# +# 1. **In-place mutation of ONE key.** Never `_record()`, never a whole-record replace: a stamp +# that rebuilt the record would reset the salt/hash (locking the user out) or the epoch +# (silently un-revoking every cookie ever minted for them). `set_password`'s docstring +# explains why that class of bug is worth naming out loud. +# +# 2. **SYNCHRONOUS FLUSH — deliberately NOT the async path, and this reverses my first draft.** +# `store.update(flush='async')` rebases on the PROCESS CACHE once a key is `_owned` +# (`core/store.py:284`) and its worker uploads that whole cached blob. For a +# table-workspace key, written by one process, that is exactly right. For `users` it is a +# silent-revert machine: tenant #0's Streamlit host writes the SAME file, so an API process +# holding a cache from an hour ago would, on its next stamp, upload a blob in which a +# password rotation or a deactivation performed in the other host simply never happened. +# A telemetry stamp must not be able to resurrect a disabled account. `flush='sync'` does a +# FRESH strict read inside the store's lock and then uploads, which is the same discipline +# every other `users` writer already uses. +# +# 3. **OFF THE REQUEST THREAD, so rule 2 costs nothing.** A sync commit is a hub round-trip, and +# neither a sign-in nor a random request an hour later should wait for it. Each stamp runs on +# a short-lived daemon thread; `flush_stamps()` is how a test or a shutdown waits for them. +# Everything is fail-silent: a stamp is telemetry and may never turn a good login into a +# failed one — the plane shows an honest "never" instead. +def _now_iso(): + import datetime as _dt + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec='seconds') + + +#: Live stamp threads, so `flush_stamps()` can join them. Bounded by construction — one thread per +#: stamp, and a stamp is at most one per login plus one per account per hour per process. +_STAMPS = [] +_STAMPS_LOCK = __import__('threading').Lock() + + +def _stamp(username, fields): + """Merge `fields` into ONE account record, on a background thread, with a fresh read.""" + username = (username or '').strip().lower() + if not username or not fields: + return None + + def _set(reg): + u = reg.get(username) + if isinstance(u, dict): + u.update(fields) + return reg + + def _work(): + try: + store.update('users', _set) # sync: fresh strict read + blocking upload + except Exception: + pass # a lost stamp is a lost stamp, never an error + import threading + t = threading.Thread(target=_work, daemon=True, name=f'user-stamp:{username}') + with _STAMPS_LOCK: + _STAMPS[:] = [x for x in _STAMPS if x.is_alive()] + _STAMPS.append(t) + t.start() + return t + + +def flush_stamps(timeout=10.0): + """Block until outstanding stamp writes have been applied. For gates and shutdown hooks — + the app never needs it, exactly like `core.store.flush`.""" + with _STAMPS_LOCK: + pending = list(_STAMPS) + for t in pending: + t.join(timeout=timeout) + return all(not t.is_alive() for t in pending) + + +def touch_login(username, when=None): + """Stamp `last_login` (ISO-8601, UTC, OFFSET-BEARING) on a SUCCESSFUL login. + + `username` must be the RESOLVED account key, not what the person typed: `verify()` accepts an + email address and resolves it to the registry key, so stamping the typed identifier would + write a stamp onto a key that does not exist and create a phantom account in the registry. + + `last_active` rides along — signing in IS activity, and setting both here means the plane's + two columns agree the moment somebody logs in rather than an hour later. + """ + stamp = when or _now_iso() + return _stamp(username, {'last_login': stamp, 'last_active': stamp}) + + +def touch_active(username, when=None): + """Stamp `last_active` — "this session did something". Throttled BY THE CALLER (`deps.py` + holds a process-local last-seen map), so this is not a store round-trip per request.""" + return _stamp(username, {'last_active': when or _now_iso()}) + + +def set_access(username, role=None, bus=None, modules=None, agent=None, email=None, + name=None, perms=None): + """Update access fields. agent/email: pass '' to clear, None to leave unchanged — + the user↔agent link scopes the Customer List page / digests to that agent's book. + + `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all: + a display name could previously only be changed by re-creating the record through + `create_user`, i.e. by also resetting the password (and, before the fix above, the session + epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + if name is not None: + u['name'] = name + if role is not None: + u['role'] = role + if bus is not None: + u['bus'] = bus + if modules is not None: + u['modules'] = modules + if agent is not None: + u['agent'] = agent or None + if email is not None: + u['email'] = email or None + if perms is not None: + # Wave 15 C-PERM. Writing perms MIGRATES the record: the marker goes on in the + # same read-modify-write, so a record can never end up with one and not the + # other (see `_record`). Whole-block replace, matching the PUT route's shape — + # a merge would make "remove this restriction" unexpressible. + u['perms'] = perms + u['perms_v'] = PERMS_VERSION + return reg + store.update('users', _set) + + +def allowed_bus_labels(user): + """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no + 'All', so the other BU is never reachable); multiple -> All + each.""" + bus = (user or {}).get('bus', 'all') + if bus == 'all': + return ['All', 'Fisch', 'Royal'] + labels = [BU_LABELS[b] for b in bus if b in BU_LABELS] + if not labels: + return ['All', 'Fisch', 'Royal'] + return (['All'] + labels) if len(labels) > 1 else labels + + +def assignable_people(tenant=None): + """Display names for `user`-typed overlay columns — the tenant's ACTIVE accounts. + + Moved from app.py (2026-07-31) so both hosts serve the same choices. Resolved on every + call rather than persisted with the column: a snapshot would keep offering people who + have left and never offer people who joined. Deactivated accounts are excluded; a value + already stored on a row is untouched — history should still say who owned something. + + Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry + is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a + cross-tenant name leak. + + ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and + return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was + written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at + EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform + roster one forgotten kwarg away, with no error to notice + ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing). + ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass + `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate + changes — and a future caller that forgets gets an empty picker somebody notices instead of a + leak nobody does. + """ + return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)}) + + +def set_avatar(username, data_url): + """Set (or clear, with None/'') the user's profile photo — a data URL (wave 14 C-AVATAR, + [[loopable-wave14-split]] item 11). Stored VERBATIM; the API route owns validation (mime + + decoded size) because this value is served back to every grid session. Cleared by removing + the key, so records without a photo keep their pre-wave shape.""" + username = (username or '').strip().lower() + + def _set(reg): + u = reg.get(username) + if u: + if data_url: + u['avatar'] = str(data_url) + else: + u.pop('avatar', None) + return reg + store.update('users', _set) + + +def avatar_map(tenant=None): + """{display name -> avatar data URL} for ACTIVE accounts with a photo — the companion of + `assignable_people()`, keyed by the SAME vocabulary: a `user` cell stores the display + name, so the display name is the only join a renderer has. Two active accounts sharing a + display name share one option; the first WITH a photo wins the key rather than a coin + flip deciding whether the option has a face. `tenant` scopes it exactly as + `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed + blank, which both take from the ONE resolver below rather than each writing `if want and …`. + """ + out = {} + for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]): + av = u.get('avatar') + nm = str(u.get('name') or username) + if av and nm not in out: + out[nm] = str(av) + return out + + +def _tenant_accounts(tenant): + """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED. + + ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice, + identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a + blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of + them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of + the same predicate and is lane C's to fold in. + + ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a + pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public` + normalises it the same way. What changed is the WANT side — a blank want is now a refusal + rather than a wildcard. + """ + want = str(tenant or '').strip().lower() + if not want: + # Not an error and not everything: nobody. See `assignable_people`'s note — the one + # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6. + return [] + try: + reg = registry() or {} + except Exception: + return [] + return [(n, u) for n, u in reg.items() + if isinstance(u, dict) and u.get('active') is not False + and str(u.get('tenant') or 'royal-imports').strip().lower() == want] diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index 2892c73329ad239d426a7a702f749413ad6a9fe9..49e0de5c77667299b3534f7a998832c9d8607a17 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -1,1042 +1,1133 @@ -"""harness/store.py — the tenant data store (OM-1, 2026-07-11). - -Incremental Odoo → DuckDB mirror: the local analytical store that saved views/dashboards (OM-3) -and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC -per question. -Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1. - -Design rules: - - The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion, - confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) — one source of truth. - - Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by - id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to - kill/re-run (upserts are idempotent). - - validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the - cent) — the store never validates itself. Odoo hard-deletes (unlink) don't move write_date; - the count checks are the drift detector for those. - - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo. -""" -import datetime as dt -import json -import os -import re -import sys -import time -from pathlib import Path - -import duckdb - -import core.odoo as O - -_STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store" - -# ⚠ MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at -# CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file. -# EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit, -# `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch — -# see its docstring for why a bare assignment is not. -# The DEFAULT is unchanged: /data/store/royal.duckdb. -DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb")) - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# ⭐ DEBT D-10 (wave 24) — THE CONNECTOR PAUSE REACHES THE MEASURE MIRROR. -# -# WHAT THE BUG WAS. Pausing a tenant's Odoo connector froze the CUSTOMER POOL (DEBT-2 built that: -# `routes_customers._pool_for` serves the pause-time snapshot) and nothing else. Every measure -# column and condition in the product is answered from THIS store, and this store kept pulling -# from Odoo the entire time a connector was paused — 12 sync passes at boot and another every -# 1800 s, hundreds of `search_read`s each. The Settings copy said so out loud ("measures not -# already computed may still reach the source until their cutover") and that sentence was the -# debt row. -# -# ⛔ WHY A PROBE AND NOT AN IMPORT. The pause flag lives in the tenant's store bucket, which is -# `harness.runtime`'s to read — and `runtime` already imports THIS module (`_ds.DB_PATH`), so an -# import back would be a cycle. The harness installs the probe at its own import instead. Nothing -# else changes: with no probe installed (every gate, every ops script, `sync_runner`) the answer -# is False and this module behaves exactly as it did. -# -# ⚠ IT FAILS TOWARDS "NOT PAUSED", deliberately and consistently with the shipped decision one -# layer up (`routes_keychain.odoo_paused`: "an unreachable flags bucket must never freeze a live -# surface"). The alternative — assume paused when the answer cannot be resolved — turns any -# transient store glitch into a mirror that silently stops advancing, which is the failure mode -# nobody notices for a week. Two copies of that policy would be worse than one; this is the same -# one. -_paused_probe = None - - -def set_paused_probe(fn): - """Install the callable that answers 'is the connector paused for the tenant whose store this - process has open?'. `harness.runtime` installs the real one; pass None to remove it.""" - global _paused_probe - _paused_probe = fn - - -def source_paused(): - """True when this store's tenant has its Odoo connector paused. Never raises.""" - try: - return bool(_paused_probe and _paused_probe()) - except Exception: - return False - - -#: What a sync entry point answers instead of reaching Odoo. A PHASE WORD, not an exception and -#: not a silent zero-row success: `status()` keeps whatever the last real pass wrote, so the -#: mirror still reports the age it genuinely has ([[no-unverifiable-aggregates]] — a paused -#: mirror that reported itself as freshly synced would be the unverifiable claim). -PAUSED_PHASE = "paused" - - -def _paused_notice(what, log): - # Printed as well as logged, and that is deliberate: `api/main.py` passes a swallowing - # `log=lambda *a, **k: None`, so the log-only version of this line would never be seen by - # the one caller that runs it unattended every 30 minutes. - msg = (f"[datastore] {what} skipped - the tenant's Odoo connector is PAUSED; the mirror is " - f"serving its pause-time state. Resume it under Settings > Connectors.") - try: - log(msg) - except Exception: - pass - print(msg) - - -def path_for(tenant_key): - """The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical - `royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename. - - File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only - boundary that holds is the operating system's — one file, one tenant. - """ - key = str(tenant_key or "").strip().lower() - if key in ("", "royal-imports"): - return DB_PATH if key == "" else Path( - os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb")) - safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60] - return _STORE_DIR / f"{safe}.duckdb" - - -def use_path(path): - """Repoint the store at `path`, CLOSING the process singleton first. - - ⚠ A bare `datastore.DB_PATH = …` is not enough and is the more dangerous half of this - operation. `_instance()` caches ONE open connection for the life of the process, so after a - plain reassignment every read keeps being served from the PREVIOUSLY opened file — a silent - cross-tenant read, returning real rows that belong to somebody else. Closing the connection - and clearing the readiness memo (which is also per-file) is what makes the switch honest. - - ⚠ AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a - cursor in a `threading.local`: this function can only clear the CALLING thread's, and - `ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was - closed out from under it in some cases — so another thread would keep answering from the old - file with no error to notice. The generation counter below is what closes that: `ro_con` - compares the generation its cursor was opened under and reopens when it has moved, which is - a check no individual thread can forget to do. - - Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process. - NOT a per-request operation — one process serves one analytical store at a time, which is why - `harness.runtime` hands out the PATH rather than switching the global underneath a request. - """ - global DB_PATH - with _INSTANCE_LOCK: - con = _INSTANCE.get("con") - if con is not None: - try: - con.close() - except Exception: - pass - _INSTANCE["con"] = None - _RO_TLS.__dict__.pop("cur", None) - _READY["ok"] = False - _GENERATION[0] += 1 # every other thread's cached cursor is now stale - DB_PATH = Path(path) - return DB_PATH - -# Entity specs: Odoo model → store table. m2o fields land as _id + _name. -# 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule). -ENTITIES = { - "sale_order": { - "model": "sale.order", "archivable": False, - "fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state", - "amount_untaxed", "invoice_status", "write_date"], - }, - "sale_order_line": { - "model": "sale.order.line", "archivable": False, - "fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty", - "price_subtotal", "margin", "purchase_price", "write_date"], - }, - "res_partner": { - "model": "res.partner", "archivable": True, - # `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales - # AGENT and an internal SALESPERSON who merely carries commission lines — the commission - # table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent") - # marks agents who came from staff; the owner ruling on whether those count as agents is - # OUTSTANDING, so both flags are synced and neither reading is baked in. - # ⭐ WAVE 29 (W29-T54, amendment A4). `customer_rank` is the ONLY predicate that - # distinguishes "a partner Odoo has TRANSACTED with" (today's population, 2,465) from - # "a customer" (3,614 with `rank>0 active`) — a 1,149-partner drop, the same join-drop - # class as item 22's SKU pool. It cannot be derived from the document tables, because - # the missing partners are precisely the ones with no document. - # ⛔ It is synced HERE rather than read from live Odoo because `read_customers` also runs - # at BOOT off a hydrated snapshot, where Odoo may be unreachable — sourcing it live would - # make the population "whatever source answered this time", changing size with the network. - # ⭐ WAVE 30 (session E's ask, W30-T33). `street`/`street2`/`zip` are the POSTAL half of - # an address the mirror has never carried: it holds `city`, `state_id` and `country_id` - # and stops there, so `odoo_relational.read_customers` — which reads only the mirror — - # cannot serve a full address at all, however the column is declared. - # ⛔ DECLARING THEM DOES NOT FILL THEM, and the difference is a whole sync: an existing - # mirror ALTERs the columns in on the next `sync_all()` and leaves them NULL until the - # backfill below has walked every partner (a partner's `write_date` did not move because - # WE changed the schema). Every projection degrades a missing/NULL column to blank rather - # than raising, so declaring early is safe — it is just not yet true. - # ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS - # ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors' - # etc."* He was right, and the census - # (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how - # much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76 - # on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here - # was measured non-empty on the live population first — none is added on the strength of - # "Odoo probably has that", which is how a schema grows columns nobody can fill. - # ⚠ THE MIRROR IS THE CEILING, NOT THE GRID. Every reader in `odoo_relational.py` reads - # THIS file, so a column added to a `*_fields()` contract with no entry here renders as a - # blank column forever — which is D-165's shape exactly, and the reason the widening starts - # on this line rather than on the grid. - "fields": ["name", "team_id", "city", "state_id", "country_id", "active", - "agent", "salesman_as_agent", "customer_rank", - "street", "street2", "zip", - "email", "phone", "mobile", "website", "vat", "ref", - "write_date"], - # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next - # sync_all() instead of needing a reseed. - # ⛔ D-165 IS ABOUT THIS LIST, AND THE CENSUS SETTLED WHICH HALF IS WRONG: `street`, - # `street2` and `zip` are `store=True` on `res.partner` and POPULATED in live Odoo — they - # are declared here and still NULL in the mirror purely because a backfill pass has not - # walked every partner (changing OUR schema does not move a partner's `write_date`, so an - # incremental sync never revisits them). Declaring is not filling; the fix is a run. - "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28 - "customer_rank", # added 2026-08-11 (W29-T54) - "street", "street2", "zip", # added 2026-08-12 (W30-T33) - "email", "phone", "mobile", # added 2026-08-14 (W33-T48) - "website", "vat", "ref"], - # m2m fields land as _id (the FIRST linked id — the Customers-module convention: - # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a - # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be - # indistinguishable from an m2o (id, name) pair in _flatten. - "m2m": ["agent_ids"], - }, - "product_product": { - "model": "product.product", "archivable": True, - "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active", - "write_date"], - }, - "account_move": { - "model": "account.move", "archivable": False, - # ⭐ `invoice_origin` added 2026-08-09 (wave 28, DEBT D-88) — the ORDER NAME an invoice was - # raised from ("S12345"), which is the only single stored key that joins an invoice back - # to its order. - # ⛔ WITHOUT IT THAT JOIN IS TWO HOPS, and the second one is expensive: invoice -> - # account_move_line -> `sale_line_ids` (the m2m BU bridge) -> sale_order_line -> - # sale_order, across 963,783 move lines. Wave 27 wanted an order->invoice preset link and - # had no single column to derive it from, so the link was not built at all. - # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text into it — a manual invoice can hold - # anything, and a merged one can hold several origins space-separated. So it is a JOIN - # HINT, and whatever consumes it matches against the order names that actually exist - # rather than trusting the string. The m2m route above stays the authority for BU - # attribution; this does not replace it. - "fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due", - "partner_id", "amount_untaxed_signed", "amount_residual_signed", - "payment_state", "invoice_origin", "write_date"], - # Existing rows never re-sync on their own (WE moved the schema; their write_date did - # not), so without this the column ALTERs in and stays NULL forever — indistinguishable - # from "this invoice genuinely has no origin". Declared, re-entrant, checkpointed. - "backfill_fields": ["invoice_origin"], # added 2026-08-09 - }, - "account_move_line": { - "model": "account.move.line", "archivable": False, - # display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the - # invoice-LINE grain is queryable (topic `invoice_lines`). - # ⚠ display_type values here are 'product', 'cogs', 'payment_term', 'line_note', - # 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones - # (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured - # 2026-07-29, omitting it does not move revenue (non-product lines carry - # price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any - # count-of-lines measure. - "fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit", - "balance", "display_type", "move_type", "parent_state", "price_subtotal", - "product_id", "write_date"], - "backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal", - "product_id"], # added 2026-07-28 - # the BU bridge: account.move carries NO business unit (every invoice sits on team 1 — - # see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable - # through the originating sale order. sale_line_id (the FIRST linked sale line) is - # lossless here: ZERO 2025 invoice lines span more than one team (measured). - "m2m": ["sale_line_ids"], - }, - "account_invoice_line_agent": { - # The OCA sale-commission module: one row per (invoice line × agent). This is the SECOND - # agent source — per-line commission attribution — and it names a different population - # than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`. - # ⚠ TWO similarly-named FKs, do not confuse them: - # object_id -> account_move_line (THE grain; join on this) - # invoice_id -> account_move (the parent document) - "model": "account.invoice.line.agent", "archivable": False, - "fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id", - "invoice_date", "settled", "write_date"], - }, - "account_account": { - # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 — - # a ('active','in',...) domain 500s); treat as non-archivable. - "model": "account.account", "archivable": False, - "fields": ["code", "name", "account_type", "write_date"], - }, -} - -BATCH = 2000 - -# Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the -# column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for -# "empty" and for "boolean false", so a bool field missing from here silently becomes NULL — and -# for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON -# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably -# from "unknown". -BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free") - -# ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence -# shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec. -# `_cols` types anything it does not recognise as VARCHAR, so `customer_rank` landed as text, the -# mirror populated correctly, every row looked right — and `odoo_relational`'s own predicate -# `customer_rank > 0` died with `Binder Error: Cannot compare values of type VARCHAR and type -# INTEGER_LITERAL`. ⛔ THE COLUMN WAS PRESENT AND THE DATA WAS CORRECT; only its TYPE was wrong, -# which is why "is the field in the mirror?" answered yes and the feature still could not run. -# ⇒ A new numeric field belongs HERE, exactly as a new boolean belongs in BOOL_FIELDS above. -INT_FIELDS = ("customer_rank", "supplier_rank") - - -import threading as _thr - -_RO_TLS = _thr.local() -_READY = {"ok": False} -_INSTANCE = {"con": None} -_INSTANCE_LOCK = _thr.Lock() -#: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with -#: the generation it was opened under, so a thread that never called `use_path` still cannot keep -#: reading the previous tenant's file — see `ro_con`. A plain "is the cursor alive?" probe cannot -#: answer that question, which is why this counter exists rather than more probing. -_GENERATION = [0] - - -def mark_ready(): - """In-process signal that the store is fully synced (every entity live). The sync writer calls - this after an all-live pass so a reader never has to (re)discover usability by touching the - file. Belt-and-suspenders alongside the shared-instance model below.""" - _READY["ok"] = True - - -def _instance(): - """THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the - sync writer uses it too — DuckDB serves concurrent reads + a single writer from ONE instance - via MVCC, so a read NEVER contends with a sync pass for the file lock. - - This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent - read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection - to same database file with a different configuration than existing connections'); a cursor on - the shared instance does not. Openers must ensure the file is fully present first — ensure_seed - writes atomically (os.replace) and every read path guards on DB_PATH.exists() — so the instance - never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole - life, so a SEPARATE process — e.g. mcp_server.py — cannot open the same store file concurrently; - in production only the app runs, and that was already true during any sync pass.)""" - con = _INSTANCE["con"] - if con is not None: - return con - with _INSTANCE_LOCK: - if _INSTANCE["con"] is None: - DB_PATH.parent.mkdir(parents=True, exist_ok=True) - _INSTANCE["con"] = duckdb.connect(str(DB_PATH)) - return _INSTANCE["con"] - - -def ro_cursor(): - """A read cursor on the shared instance — the contention-free way to read the store.""" - return _instance().cursor() - - -def ready(): - """True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on - this: a mid-backfill store would return PARTIAL totals silently — worse than an error. Positive - results cache for the process (backfill never regresses). Reads via a shared-instance cursor, - so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight.""" - if _READY["ok"]: - return True - if not DB_PATH.exists(): - return False - try: - cur = ro_cursor() - try: - phases = {r[0]: r[1] for r in - cur.execute("SELECT entity, phase FROM _sync_state").fetchall()} - finally: - cur.close() - except Exception: - return False - if all(phases.get(k) == "live" for k in ENTITIES): - _READY["ok"] = True - return True - return False - - -def ro_con(): - """Per-thread cached read CURSOR on the shared instance for page/module store reads (the - open cost dominates repeated small queries — the Expenses retrofit measured 7.7s→1.0s). No - lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES - until ready() — callers either fall back to live reads (modules) or relay a readable message - (Analyst). - - ⚠ THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness - probe below asks "does this cursor still work?", which is a different question from "is this - cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches - tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` — - and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no - exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is - discarded here, which is the only place every thread is guaranteed to pass through.""" - cur = getattr(_RO_TLS, "cur", None) - if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]: - try: - cur.close() - except Exception: - pass - cur, _RO_TLS.cur = None, None # the store moved underneath this thread - if cur is not None: - try: - cur.execute("SELECT 1") # cheap liveness probe - return cur - except Exception: - _RO_TLS.cur = None # stale → reopen below - if not ready(): - raise RuntimeError("the tenant data store is completing its FIRST sync on this " - "deployment — dashboards work meanwhile (live reads); retry " - "store queries in a few minutes") - cur = ro_cursor() - _RO_TLS.cur = cur - _RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to - return cur - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# ⭐ WAVE 30 / OWNER RULING R6 + R7 — THE WINDOW. Read this before touching `window()`. -# -# R6, verbatim: *"there is no cap in how many data from the API source (as long as its from a -# connected source like Odoo) that can be pulled into the app."* R7 says how: a connected grid -# reads THROUGH this mirror instead of copying rows into the one `user_tables` document, which -# `MAX_ROWS = 60_000` bounds. The mirror is already uncapped — 971,034 GL lines live in this file -# today — so "no cap" is not a bigger number, it is a different mechanism: serve a SLICE and count -# the whole. -# -# ⛔⛔ `total` IS A `SELECT count(*)` OVER THE SAME PREDICATE, NEVER `len(rows)`. A window whose -# count is its own length is a fabricated aggregate that reads as authoritative — the class this -# repo has paid for twice ([[no-unverifiable-aggregates]], [[one-question-two-normalizers]]: a -# display predicate and a fold predicate answering one question differently). The two statements -# below are built from ONE `where` + ONE `params` tuple for exactly that reason; they cannot drift -# apart without deleting a line. -# -# ⛔ AND THE PREDICATE PUSHES DOWN. `where` is compiled by `harness/filter_sql.py` — the same -# compiler the client's filter engine is held in step with — so a filter matching rows outside the -# loaded window still COUNTS them. A caller that filters the returned list instead has silently -# asked "how many of the 200 rows in memory match" about a 971,034-row table. -# -# ⚠ `order_by` IS REQUIRED IN PRACTICE AND DEFAULTED HERE. Two pages of an UNORDERED window are -# not guaranteed to partition the table: DuckDB may legally return a row on page 1 and again on -# page 2, and the user sees a duplicate with no error anywhere. The default is the physical -# rowid-ish `1` only when a caller genuinely has no key; every real caller passes one. -# -# ⚠ TRUST BOUNDARY. `table`, `select`, `where` and `order_by` are SQL we author (a spec row, or -# `filter_sql`'s output over a whitelisted column map). Every VALUE is bound through `params`. -# `table` is additionally shape-checked below — not because a caller is hostile, but because a -# typo'd identifier interpolated into two statements is worth one cheap assertion. - -#: The most rows ONE request may carry back. Not a cap on the data — every row is reachable by -#: paging and `total` always tells the truth about how many there are — but a bound on the memory -#: a single response can cost. R6's second sentence applies: a caller that asks for more is CLAMPED -#: and the clamp is REPORTED (`routes_odoo_tables` turns it into a `limits` entry), never silent. -WINDOW_MAX = 5_000 - -#: What a caller gets when it names no window at all. Matches `semantic.store_rows`' own default so -#: the two windowed readers in this codebase do not disagree about what "a page" means. -WINDOW_DEFAULT = 200 - -_IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - - -def window(table=None, select=None, where=None, params=(), order_by=None, - offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None): - """One page of a mirror table PLUS the true count of everything the same predicate matches. - - Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int, - "limit": int, "clamped": bool}` — `rows` is the slice, `total` is the population. - - `select` is the projection expression list ("id, name, amount_untaxed"); `where` is a WHERE - body with `?` placeholders; `params` binds them, and is used by BOTH statements. - - ⚠ `table` and `from_sql` are the SAME argument wearing two trust levels, and they are separate - names on purpose. `table` is an identifier and is SHAPE-CHECKED; `from_sql` is a whole FROM - body — a parenthesised subquery with an alias — which cannot be checked at all. Two of the - eight connected grids need one (`read_customers` and `read_agents` are single statements whose - population is a SQL `UNION`), so the capability has to exist; naming it distinctly means a - reviewer can grep for every caller that hands over raw FROM SQL instead of inferring it. - """ - if bool(table) == bool(from_sql): - raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`") - if from_sql: - name = str(from_sql).strip() - else: - name = str(table or "").strip() - if not _IDENT_OK.match(name): - raise ValueError(f"datastore.window: {table!r} is not a table identifier") - proj = str(select or "").strip() - if not proj: - raise ValueError("datastore.window: a projection is required — there is no implicit *") - try: - offset = max(0, int(offset or 0)) - except (TypeError, ValueError): - offset = 0 - try: - limit = int(limit if limit is not None else WINDOW_DEFAULT) - except (TypeError, ValueError): - limit = WINDOW_DEFAULT - clamped = limit > WINDOW_MAX or limit < 1 - limit = min(max(limit, 1), WINDOW_MAX) - - params = tuple(params or ()) - pred = str(where or "").strip() - tail = f" WHERE {pred}" if pred else "" - order = str(order_by or "").strip() or "1" - cur = cur if cur is not None else ro_con() - - # ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate: - # if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than - # quietly returning a good count beside a broken page. - total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0) - got = cur.execute( - f"SELECT {proj} FROM {name}{tail} ORDER BY {order} LIMIT {int(limit)} OFFSET {int(offset)}", - params).fetchall() - cols = [d[0] for d in (cur.description or [])] - return {"rows": [tuple(r) for r in got], "columns": cols, - "total": total, "offset": offset, "limit": limit, "clamped": clamped} - - -def columns_of(table, cur=None): - """The column names a mirror table actually has, lowercased. - - ⚠ A mirror can be `ready()` and still be MISSING COLUMNS: `ready()` reads entity PHASES, while - `_ensure_columns`/`_backfill_columns` checkpoint separately. Asking before projecting is what - stops a DuckDB `BinderException: Referenced column … not found` reaching a user as a bare 500 - (it already cost one live, which is why `odoo_relational.columns` exists on the app side). - """ - name = str(table or "").strip() - if not _IDENT_OK.match(name): - raise ValueError(f"datastore.columns_of: {table!r} is not a table identifier") - cur = cur if cur is not None else ro_con() - try: - rows = cur.execute(f"PRAGMA table_info('{name}')").fetchall() - except Exception: # noqa: BLE001 - return set() - return {str(r[1]).lower() for r in rows} - - -SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data") - - -def ensure_seed(): - """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store - from the PRIVATE dataset's seed snapshot — Analyst-ready in ~a minute instead of a 20-40 min - XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores - the pilot views/routines when absent. Fail-quiet: no seed/token → normal backfill.""" - import shutil - if DB_PATH.exists(): - return False - tok = os.environ.get("HF_TOKEN") - if not tok: - return False - try: - from huggingface_hub import hf_hub_download - DB_PATH.parent.mkdir(parents=True, exist_ok=True) - p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb", - repo_type="dataset", token=tok) - # Atomic install: copy to a temp path then os.replace — a reader that guards on - # DB_PATH.exists() must never bind a half-written file into the shared instance. - tmp = DB_PATH.with_name(DB_PATH.name + ".tmp") - shutil.copyfile(p, tmp) - os.replace(tmp, DB_PATH) - for fn in ("views.json", "routines.json"): - tgt = DB_PATH.parent / fn - if not tgt.exists(): - try: - q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}", - repo_type="dataset", token=tok) - shutil.copyfile(q, tgt) - except Exception: - pass - return True - except Exception: - return False - - -def connect(): - """A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a - cursor so callers' con.close() frees the cursor without closing the process-wide instance; - one writer cursor is active at a time (the sync thread is sequential), readers use others.""" - cur = _instance().cursor() - cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state ( - entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, - cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") - return cur - - -def _flatten(rec, fields, m2m=()): - out = {"id": rec["id"]} - for f in m2m: - v = rec.get(f) or [] - base = f.removesuffix("_ids") - out[f"{base}_id"] = v[0] if v else None - out[f] = json.dumps(list(v)) if v else None - for f in fields: - v = rec.get(f) - if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int): - out[f"{f.removesuffix('_id')}_id"] = v[0] - out[f"{f.removesuffix('_id')}_name"] = str(v[1]) - elif isinstance(v, (list, tuple)): - out[f] = json.dumps(v) - elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields - out[f] = None - else: - out[f] = v - return out - - -def _cols(spec): - cols = ["id BIGINT PRIMARY KEY"] - for f in spec["fields"]: - base = f.removesuffix("_id") - if f.endswith("_id"): - cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"] - elif f in BOOL_FIELDS: - cols.append(f"{f} BOOLEAN") - elif f in INT_FIELDS: - cols.append(f"{f} BIGINT") - elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed", - "price_subtotal", "margin", "purchase_price", "product_uom_qty", - "standard_price", "debit", "credit", "balance", "amount"): - cols.append(f"{f} DOUBLE") - else: - cols.append(f"{f} VARCHAR") - for f in spec.get("m2m") or []: - base = f.removesuffix("_ids") - cols += [f"{base}_id BIGINT", f"{f} VARCHAR"] - return cols - - -def _ensure_table(con, key, spec): - con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})") - - -def _ensure_columns(con, key, spec): - """Schema evolution: existing stores/seeds predate columns a newer spec introduces — - CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names.""" - have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()} - added = [] - for c in _cols(spec): - name = c.split()[0] - if name not in have: - con.execute(f"ALTER TABLE {key} ADD COLUMN {c}") - added.append(name) - return added - - -def _field_cols(f): - """The store column(s) one spec field materialises into (m2o fields become _id + _name).""" - if f.endswith("_id"): - base = f.removesuffix("_id") - return [f"{base}_id", f"{base}_name"] - return [f] - - -def _backfill_columns(con, key, spec, log=print): - """Scalar/m2o schema evolution — the twin of _backfill_m2m, and for the same reason. - - _ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema; - their write_date didn't move), so a newly added field stays NULL forever — silently, and NULL - is indistinguishable from a legitimately empty value. That is how a store starts answering - 'no rows match' instead of erroring. Re-pull just the affected fields for every row, - paginated by id, marked durably so a crash retries rather than declaring completion. - - The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns - did _ensure_columns just add" — that inference is wrong on the second run, when the ALTER has - already happened and the list comes back empty while the data is still missing. Declaring it - makes the migration re-entrant and reviewable. Leave the entry in place after it completes; - the durable marker, not the absence of the declaration, is what stops it re-running. - """ - fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"] - if not fields: - return - marker = f"{key}.cols.{','.join(sorted(fields))}" - st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?", - [marker]).fetchone() - if st and st[0] == "done": - return - # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses - # an hour of work to one dropped connection gets skipped by the next person under time - # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an - # interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring - # completion it never reached. - last, n = (st[1] or 0, st[2] or 0) if st else (0, 0) - if last: - log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)") - cols = [c for f in fields for c in _field_cols(f)] - setter = ", ".join(f"{c}=?" for c in cols) - while True: - recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], - fields, limit=5000, order="id asc") - if not recs: - break - rows = [_flatten(r, fields) for r in recs] - con.executemany(f"UPDATE {key} SET {setter} WHERE id=?", - [[r.get(c) for c in cols] + [r["id"]] for r in rows]) - last, n = recs[-1]["id"], n + len(recs) - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) - log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)") - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) - log(f" {key}: column backfill complete -> {n:,} rows") - - -def _backfill_m2m(con, key, spec, log=print): - """One-time targeted backfill after a schema migration: existing rows never re-sync (their - write_date didn't move when WE added a column), so pull every record where the m2m field is - SET and update in place. Rows with the field empty keep NULL — correct and free. Completion - is a durable _sync_state marker ('.m2m.') — NOT 'column just added': a crash - between the ALTER and this backfill (seen live: the SSL-failed first run) must retry.""" - for f in spec.get("m2m") or []: - marker = f"{key}.m2m.{f}" - if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone(): - continue - base = f.removesuffix("_ids") - # PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line - # has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone - # missing with no error — a partial backfill that reads as a complete one (the - # [[no-unverifiable-aggregates]] no-silent-caps rule). - rows, last, page = [], 0, 20000 - while True: - recs = O.search_read(spec["model"], - _base_domain(spec) + [(f, "!=", False), ("id", ">", last)], - [f], limit=page, order="id asc") - if not recs: - break - rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)] - last = recs[-1]["id"] - log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)") - if rows: - con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows) - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - [marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")]) - log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated") - - -def _upsert(con, key, spec, recs): - if not recs: - return - rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs] - colnames = [c.split()[0] for c in _cols(spec)] - ids = [r["id"] for r in rows] - con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})") - con.executemany( - f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})", - [[r.get(c) for c in colnames] for r in rows]) - - -def _state(con, key): - row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?", - [key]).fetchone() - return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None - - -def _save_state(con, key, phase, last_id, cursor_wd): - n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] - con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""", - [key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")]) - - -def _base_domain(spec): - return [("active", "in", [True, False])] if spec["archivable"] else [] - - -def sync_entity(key, max_batches=200, batch=BATCH, log=print): - """One bounded, resumable sync pass for an entity. Backfill (by id) → live (by write_date). - Returns (phase, pulled_rows). - - ⭐ D-10: refuses BEFORE opening a cursor or touching Odoo while the connector is paused. The - guard is here — the innermost function that reads from `O` — rather than only in `sync_all`, - so a caller that syncs ONE entity is covered by construction instead of by remembering to - add the same check. `sync_all` short-circuits too, but only to avoid eight identical notices. - """ - if source_paused(): - _paused_notice(f"sync of {key}", log) - return PAUSED_PHASE, 0 - spec = ENTITIES[key] - con = connect() - _ensure_table(con, key, spec) - _ensure_columns(con, key, spec) - _backfill_columns(con, key, spec, log=log) - _backfill_m2m(con, key, spec, log=log) - st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0} - fields = spec["fields"] + list(spec.get("m2m") or []) - pulled = 0 - try: - if st["phase"] == "backfill": - last_id = st["last_id"] or 0 - for i in range(max_batches): - dom = _base_domain(spec) + [("id", ">", last_id)] - recs = O.search_read(spec["model"], dom, ["id"] + fields, - limit=batch, order="id asc") - if not recs: - # backfill complete → switch to live mode anchored at max write_date seen - wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0] - _save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00") - log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode") - return "live", pulled - _upsert(con, key, spec, recs) - last_id = recs[-1]["id"] - pulled += len(recs) - _save_state(con, key, "backfill", last_id, None) - if (i + 1) % 10 == 0: - log(f" {key}: backfill …{pulled:,} rows (id≤{last_id})") - log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)") - return "backfill", pulled - # live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP - # FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE - # write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15 - # 16:35:03'; stored values carry microseconds, XML-RPC strings don't) — then the - # write_date cursor can NEVER advance and every pass re-pulls the same first page - # forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over - # write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|' in - # cursor_wd so a killed walk resumes; on completion the cursor jumps one second past - # the stamp (safe: the walk covered every row in that second, and the stamp is in the - # past — guarded by a 2-minute recency check). - raw = st["cursor_wd"] or "1970-01-01 00:00:00" - cursor, _pipe, _tie = raw.partition("|") - walking, tie_id = bool(_pipe), int(_tie or 0) - walk_max = cursor - for _ in range(max_batches): - if walking: - dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)] - recs = O.search_read(spec["model"], dom, ["id"] + fields, - limit=batch, order="id asc") - if not recs: # walk complete → advance past the stamp - try: - wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S") - if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2): - cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S") - else: - cursor = walk_max - except ValueError: - cursor = walk_max - walking = False - log(f" {key}: id-walk complete -> cursor {cursor}") - break - _upsert(con, key, spec, recs) - pulled += len(recs) - tie_id = recs[-1]["id"] - walk_max = max(walk_max, max(str(r["write_date"]) for r in recs)) - _save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}") - continue - dom = _base_domain(spec) + [("write_date", ">=", cursor)] - recs = O.search_read(spec["model"], dom, ["id"] + fields, - limit=batch, order="write_date asc, id asc") - if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor - and pulled == 0 and _already(con, key, recs[0])): - break - new_cursor = str(recs[-1]["write_date"]) - if new_cursor == cursor and len(recs) == batch: - # full batch stuck on one second: enter the id-walk from id 0 (the write_date - # ordering shuffles ids within the stamp second, so the batch just pulled is - # NOT an id prefix — restart coverage by id; upserts make the overlap free) - walking, tie_id, walk_max = True, 0, cursor - _save_state(con, key, "live", st["last_id"], f"{cursor}|0") - log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk") - continue - _upsert(con, key, spec, recs) - pulled += len(recs) - if new_cursor == cursor and len(recs) < batch: - cursor = new_cursor - break - cursor = new_cursor - # Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the - # 2026-07-15 recompute) means a killed pass otherwise re-pulls everything — - # upserts are idempotent, but the wasted RPC is real (learned 2026-07-17). - _save_state(con, key, "live", st["last_id"], cursor) - _save_state(con, key, "live", st["last_id"], - f"{cursor}|{tie_id}" if walking else cursor) - if pulled: - log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}" - + (f" walking id>{tie_id}" if walking else "") + ")") - return "live", pulled - finally: - con.close() - - -def _already(con, key, rec): - return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone()) - - -def sync_all(entities=None, max_batches=200, log=print): - # ⭐ D-10. `{}` rather than a dict of paused phases, and the caller decides why that matters: - # `api/main.py:354` loops up to 12 times until `all(phase == "live")`, and `all()` over an - # empty dict is True — so a paused store costs ONE pass instead of twelve rounds of the same - # refusal. A dict of `"paused"` phases would fail that test every time and turn the boot - # sprint into a busy-wait against a connector nobody has resumed. - if source_paused(): - _paused_notice("sync", log) - return {} - out = {} - for key in (entities or list(ENTITIES)): - t0 = time.time() - phase, pulled = sync_entity(key, max_batches=max_batches, log=log) - out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)} - # A full pass that lands every entity in live mode latches process readiness in-process — so - # readers never race the writer's lock to (re)discover the store is usable. - if entities is None and out and all(v["phase"] == "live" for v in out.values()): - mark_ready() - return out - - -def reconcile_deletes(entities=None, log=print): - """Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor - sync can never see a deletion — the count parity DETECTS the drift (by design); this repairs - it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are - cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the - app's frequent passes. - - ⭐ D-10: it pulls the FULL live id set per entity, so it is the single largest Odoo read in - this module and it runs unattended at boot and every fourth resync pass. Paused means paused. - """ - if source_paused(): - _paused_notice("delete reconcile", log) - return {} - con = connect() - try: - out = {} - for key in (entities or list(ENTITIES)): - spec = ENTITIES[key] - live_ids, last = set(), 0 - while True: - recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], - ["id"], limit=50000, order="id asc") - if not recs: - break - live_ids.update(r["id"] for r in recs) - last = recs[-1]["id"] - store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()} - dead = store_ids - live_ids - if dead: - con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)") - con.execute("DELETE FROM _dead") - con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead]) - con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)") - con.execute("DELETE FROM _dead") - n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] - con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key]) - out[key] = len(dead) - log(f" {key}: reconciled {len(dead):,} hard-deleted rows") - return out - finally: - con.close() - - -def status(): - con = connect() - try: - return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]} - for r in con.execute("SELECT * FROM _sync_state").fetchall()} - finally: - con.close() - - -# ------------------------------------------------------------------ parity (store vs LIVE Odoo) - -def _check(name, ours, live, tol=0.01): - gap = abs((ours or 0) - (live or 0)) - return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live} - - -def validate(pre=None, months=("2026-01-01", "2026-07-01")): - """Raw-fidelity parity: the store must mirror live Odoo — row counts per synced entity + - unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase. - - ⭐ D-10 REFUSES LOUDLY HERE rather than skipping quietly, and the difference matters. Every - other paused path has a correct silent answer — serve the mirror — because the mirror is the - thing being asked for. This function's ENTIRE contract is "compare me against live Odoo", so - a paused version of it has no honest result: skipping would return an empty pass, and - proceeding would make the live call the pause forbids. An operator running a reconciliation - against a paused connector has asked a question that cannot be answered, and being told so is - the only outcome that is not a lie. (Operator-invoked only — nothing in the serving path - calls it, so this raises for a person, never inside a request.) - """ - if source_paused(): - raise RuntimeError( - "the tenant's Odoo connector is PAUSED, so the store cannot be validated against " - "live Odoo - resume it under Settings > Connectors and run this again") - con = connect() - out = [] - try: - live_state = status() - d0, d1 = months - for key, spec in ENTITIES.items(): - if live_state.get(key, {}).get("phase") != "live": - continue - ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] - live = O.get_odoo().search_count(spec["model"], _base_domain(spec)) - out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0)) - if live_state.get("sale_order_line", {}).get("phase") == "live": - ours = con.execute( - "SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l " - "JOIN sale_order o ON o.id = l.order_id " - "WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0] - live = O.sum_field("sale.order.line", - [("order_id.date_order", ">=", f"{d0} 00:00:00"), - ("order_id.date_order", "<", f"{d1} 00:00:00")], - "price_subtotal") - out.append(_check(f"store Σ line revenue [{d0}..{d1}) == live (unscoped)", ours, live)) - if live_state.get("sale_order", {}).get("phase") == "live": - ours = con.execute( - "SELECT coalesce(sum(amount_untaxed),0) FROM sale_order " - "WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0] - live = O.sum_field("sale.order", - [("date_order", ">=", f"{d0} 00:00:00"), - ("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed") - out.append(_check(f"store Σ order amount_untaxed [{d0}..{d1}) == live (unscoped)", - ours, live)) - finally: - con.close() - return out - - -if __name__ == "__main__": - args = sys.argv[1:] - if args and args[0] == "status": - for k, v in status().items(): - print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})") - elif args and args[0] == "reconcile": - ents = args[1].split(",") if len(args) > 1 else None - for k, n in reconcile_deletes(ents).items(): - print(f" {k:18s} -{n:,} hard-deleted rows") - elif args and args[0] == "validate": - ok = True - for c in validate(): - mark = "OK " if c["ok"] else "XX " - ok &= c["ok"] - print(f" {mark}{c['check'][:80]} gap={c['gap']}") - print("PARITY:", "GREEN" if ok else "RED") - sys.exit(0 if ok else 1) - else: - ents = args[0].split(",") if args else None - mb = int(args[1]) if len(args) > 1 else 200 - for k, v in sync_all(ents, max_batches=mb).items(): - print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s") +"""harness/store.py — the tenant data store (OM-1, 2026-07-11). + +Incremental Odoo → DuckDB mirror: the local analytical store that saved views/dashboards (OM-3) +and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC +per question. +Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1. + +Design rules: + - The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion, + confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) — one source of truth. + - Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by + id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to + kill/re-run (upserts are idempotent). + - validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the + cent) — the store never validates itself. Odoo hard-deletes (unlink) don't move write_date; + the count checks are the drift detector for those. + - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo. +""" +import datetime as dt +import json +import os +import re +import sys +import time +from pathlib import Path + +import duckdb + +import core.odoo as O + +_STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store" + +# ⚠ MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at +# CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file. +# EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit, +# `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch — +# see its docstring for why a bare assignment is not. +# The DEFAULT is unchanged: /data/store/royal.duckdb. +DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb")) + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# ⭐ DEBT D-10 (wave 24) — THE CONNECTOR PAUSE REACHES THE MEASURE MIRROR. +# +# WHAT THE BUG WAS. Pausing a tenant's Odoo connector froze the CUSTOMER POOL (DEBT-2 built that: +# `routes_customers._pool_for` serves the pause-time snapshot) and nothing else. Every measure +# column and condition in the product is answered from THIS store, and this store kept pulling +# from Odoo the entire time a connector was paused — 12 sync passes at boot and another every +# 1800 s, hundreds of `search_read`s each. The Settings copy said so out loud ("measures not +# already computed may still reach the source until their cutover") and that sentence was the +# debt row. +# +# ⛔ WHY A PROBE AND NOT AN IMPORT. The pause flag lives in the tenant's store bucket, which is +# `harness.runtime`'s to read — and `runtime` already imports THIS module (`_ds.DB_PATH`), so an +# import back would be a cycle. The harness installs the probe at its own import instead. Nothing +# else changes: with no probe installed (every gate, every ops script, `sync_runner`) the answer +# is False and this module behaves exactly as it did. +# +# ⚠ IT FAILS TOWARDS "NOT PAUSED", deliberately and consistently with the shipped decision one +# layer up (`routes_keychain.odoo_paused`: "an unreachable flags bucket must never freeze a live +# surface"). The alternative — assume paused when the answer cannot be resolved — turns any +# transient store glitch into a mirror that silently stops advancing, which is the failure mode +# nobody notices for a week. Two copies of that policy would be worse than one; this is the same +# one. +_paused_probe = None + + +def set_paused_probe(fn): + """Install the callable that answers 'is the connector paused for the tenant whose store this + process has open?'. `harness.runtime` installs the real one; pass None to remove it.""" + global _paused_probe + _paused_probe = fn + + +def source_paused(): + """True when this store's tenant has its Odoo connector paused. Never raises.""" + try: + return bool(_paused_probe and _paused_probe()) + except Exception: + return False + + +#: What a sync entry point answers instead of reaching Odoo. A PHASE WORD, not an exception and +#: not a silent zero-row success: `status()` keeps whatever the last real pass wrote, so the +#: mirror still reports the age it genuinely has ([[no-unverifiable-aggregates]] — a paused +#: mirror that reported itself as freshly synced would be the unverifiable claim). +PAUSED_PHASE = "paused" + + +def _paused_notice(what, log): + # Printed as well as logged, and that is deliberate: `api/main.py` passes a swallowing + # `log=lambda *a, **k: None`, so the log-only version of this line would never be seen by + # the one caller that runs it unattended every 30 minutes. + msg = (f"[datastore] {what} skipped - the tenant's Odoo connector is PAUSED; the mirror is " + f"serving its pause-time state. Resume it under Settings > Connectors.") + try: + log(msg) + except Exception: + pass + print(msg) + + +def path_for(tenant_key): + """The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical + `royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename. + + File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only + boundary that holds is the operating system's — one file, one tenant. + """ + key = str(tenant_key or "").strip().lower() + if key in ("", "royal-imports"): + return DB_PATH if key == "" else Path( + os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb")) + safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60] + return _STORE_DIR / f"{safe}.duckdb" + + +def use_path(path): + """Repoint the store at `path`, CLOSING the process singleton first. + + ⚠ A bare `datastore.DB_PATH = …` is not enough and is the more dangerous half of this + operation. `_instance()` caches ONE open connection for the life of the process, so after a + plain reassignment every read keeps being served from the PREVIOUSLY opened file — a silent + cross-tenant read, returning real rows that belong to somebody else. Closing the connection + and clearing the readiness memo (which is also per-file) is what makes the switch honest. + + ⚠ AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a + cursor in a `threading.local`: this function can only clear the CALLING thread's, and + `ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was + closed out from under it in some cases — so another thread would keep answering from the old + file with no error to notice. The generation counter below is what closes that: `ro_con` + compares the generation its cursor was opened under and reopens when it has moved, which is + a check no individual thread can forget to do. + + Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process. + NOT a per-request operation — one process serves one analytical store at a time, which is why + `harness.runtime` hands out the PATH rather than switching the global underneath a request. + """ + global DB_PATH + with _INSTANCE_LOCK: + con = _INSTANCE.get("con") + if con is not None: + try: + con.close() + except Exception: + pass + _INSTANCE["con"] = None + _RO_TLS.__dict__.pop("cur", None) + _READY["ok"] = False + _GENERATION[0] += 1 # every other thread's cached cursor is now stale + DB_PATH = Path(path) + return DB_PATH + +# Entity specs: Odoo model → store table. m2o fields land as _id + _name. +# 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule). +ENTITIES = { + "sale_order": { + "model": "sale.order", "archivable": False, + "fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state", + "amount_untaxed", "invoice_status", "write_date"], + }, + "sale_order_line": { + "model": "sale.order.line", "archivable": False, + "fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty", + "price_subtotal", "margin", "purchase_price", "write_date"], + }, + "res_partner": { + "model": "res.partner", "archivable": True, + # `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales + # AGENT and an internal SALESPERSON who merely carries commission lines — the commission + # table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent") + # marks agents who came from staff; the owner ruling on whether those count as agents is + # OUTSTANDING, so both flags are synced and neither reading is baked in. + # ⭐ WAVE 29 (W29-T54, amendment A4). `customer_rank` is the ONLY predicate that + # distinguishes "a partner Odoo has TRANSACTED with" (today's population, 2,465) from + # "a customer" (3,614 with `rank>0 active`) — a 1,149-partner drop, the same join-drop + # class as item 22's SKU pool. It cannot be derived from the document tables, because + # the missing partners are precisely the ones with no document. + # ⛔ It is synced HERE rather than read from live Odoo because `read_customers` also runs + # at BOOT off a hydrated snapshot, where Odoo may be unreachable — sourcing it live would + # make the population "whatever source answered this time", changing size with the network. + # ⭐ WAVE 30 (session E's ask, W30-T33). `street`/`street2`/`zip` are the POSTAL half of + # an address the mirror has never carried: it holds `city`, `state_id` and `country_id` + # and stops there, so `odoo_relational.read_customers` — which reads only the mirror — + # cannot serve a full address at all, however the column is declared. + # ⛔ DECLARING THEM DOES NOT FILL THEM, and the difference is a whole sync: an existing + # mirror ALTERs the columns in on the next `sync_all()` and leaves them NULL until the + # backfill below has walked every partner (a partner's `write_date` did not move because + # WE changed the schema). Every projection degrades a missing/NULL column to blank rather + # than raising, so declaring early is safe — it is just not yet true. + # ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS + # ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors' + # etc."* He was right, and the census + # (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how + # much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76 + # on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here + # was measured non-empty on the live population first — none is added on the strength of + # "Odoo probably has that", which is how a schema grows columns nobody can fill. + # ⚠ THE MIRROR IS THE CEILING, NOT THE GRID. Every reader in `odoo_relational.py` reads + # THIS file, so a column added to a `*_fields()` contract with no entry here renders as a + # blank column forever — which is D-165's shape exactly, and the reason the widening starts + # on this line rather than on the grid. + "fields": ["name", "team_id", "city", "state_id", "country_id", "active", + "agent", "salesman_as_agent", "customer_rank", + "street", "street2", "zip", + "email", "phone", "mobile", "website", "vat", "ref", + "write_date"], + # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next + # sync_all() instead of needing a reseed. + # ⛔ D-165 IS ABOUT THIS LIST, AND THE CENSUS SETTLED WHICH HALF IS WRONG: `street`, + # `street2` and `zip` are `store=True` on `res.partner` and POPULATED in live Odoo — they + # are declared here and still NULL in the mirror purely because a backfill pass has not + # walked every partner (changing OUR schema does not move a partner's `write_date`, so an + # incremental sync never revisits them). Declaring is not filling; the fix is a run. + "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28 + "customer_rank", # added 2026-08-11 (W29-T54) + "street", "street2", "zip", # added 2026-08-12 (W30-T33) + "email", "phone", "mobile", # added 2026-08-14 (W33-T48) + "website", "vat", "ref"], + # m2m fields land as _id (the FIRST linked id — the Customers-module convention: + # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a + # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be + # indistinguishable from an m2o (id, name) pair in _flatten. + "m2m": ["agent_ids"], + }, + "product_product": { + "model": "product.product", "archivable": True, + "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active", + "write_date"], + }, + "account_move": { + "model": "account.move", "archivable": False, + # ⭐ `invoice_origin` added 2026-08-09 (wave 28, DEBT D-88) — the ORDER NAME an invoice was + # raised from ("S12345"), which is the only single stored key that joins an invoice back + # to its order. + # ⛔ WITHOUT IT THAT JOIN IS TWO HOPS, and the second one is expensive: invoice -> + # account_move_line -> `sale_line_ids` (the m2m BU bridge) -> sale_order_line -> + # sale_order, across 963,783 move lines. Wave 27 wanted an order->invoice preset link and + # had no single column to derive it from, so the link was not built at all. + # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text into it — a manual invoice can hold + # anything, and a merged one can hold several origins space-separated. So it is a JOIN + # HINT, and whatever consumes it matches against the order names that actually exist + # rather than trusting the string. The m2m route above stays the authority for BU + # attribution; this does not replace it. + "fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due", + "partner_id", "amount_untaxed_signed", "amount_residual_signed", + "payment_state", "invoice_origin", "write_date"], + # Existing rows never re-sync on their own (WE moved the schema; their write_date did + # not), so without this the column ALTERs in and stays NULL forever — indistinguishable + # from "this invoice genuinely has no origin". Declared, re-entrant, checkpointed. + "backfill_fields": ["invoice_origin"], # added 2026-08-09 + }, + "account_move_line": { + "model": "account.move.line", "archivable": False, + # display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the + # invoice-LINE grain is queryable (topic `invoice_lines`). + # ⚠ display_type values here are 'product', 'cogs', 'payment_term', 'line_note', + # 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones + # (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured + # 2026-07-29, omitting it does not move revenue (non-product lines carry + # price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any + # count-of-lines measure. + "fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit", + "balance", "display_type", "move_type", "parent_state", "price_subtotal", + "product_id", "write_date"], + "backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal", + "product_id"], # added 2026-07-28 + # the BU bridge: account.move carries NO business unit (every invoice sits on team 1 — + # see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable + # through the originating sale order. sale_line_id (the FIRST linked sale line) is + # lossless here: ZERO 2025 invoice lines span more than one team (measured). + "m2m": ["sale_line_ids"], + }, + "account_invoice_line_agent": { + # The OCA sale-commission module: one row per (invoice line × agent). This is the SECOND + # agent source — per-line commission attribution — and it names a different population + # than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`. + # ⚠ TWO similarly-named FKs, do not confuse them: + # object_id -> account_move_line (THE grain; join on this) + # invoice_id -> account_move (the parent document) + "model": "account.invoice.line.agent", "archivable": False, + "fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id", + "invoice_date", "settled", "write_date"], + }, + "account_account": { + # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 — + # a ('active','in',...) domain 500s); treat as non-archivable. + "model": "account.account", "archivable": False, + "fields": ["code", "name", "account_type", "write_date"], + }, + # ⭐⭐ W37-T13 (owner: stock moved IN and OUT per SKU) — MEASURED FIRST in + # `proto/P1-stock-moves.md`, and every clause below is a finding rather than a guess. + # + # ⛔ THE TWO LOCATION COLUMNS ARE BOTH LOAD-BEARING AND THIS IS THE TICKET'S HEADLINE TRAP. + # A move is an IN when it ARRIVES somewhere internal from somewhere that is not, and an OUT + # when it LEAVES somewhere internal for somewhere that is not. `internal -> internal` is + # **58% of all moves** and is NEITHER — a one-sided domain counts every one of them as BOTH, + # which roughly doubles both columns while looking entirely plausible. + # + # ⛔ `picking_code` IS NOT THE DISCRIMINATOR, and the reason is worth the line: it is + # `store=False`, so a `read_group` on it fails hard, AND `('picking_code','=',False)` returns + # 0 while 5,800 pickingless done moves exist — **the count that would have warned you also + # reads 0**. `picking_id` is stored and is carried instead, for provenance only. + # + # ⚠ `quantity_done` IS THE AGGREGATE, not `product_uom_qty`: it ties to + # `stock.move.line.qty_done` to the cent over the full window, and it is what actually moved. + # ⚠ NOT ARCHIVABLE — `stock.move` has no `active` field; a move is history and is never + # archived. Stated so the next reader does not "fix" it by adding `active in [True, False]`. + # ⚠ SIZE: 236k rows over 1,095 days (P1), about a quarter of `account_move_line`'s 963,783. + "stock_move": { + # ⛔ `optional` — see `ready()`. A NEW entity must never gate the readiness of surfaces + # that do not read it, or adding one takes the product down until it has synced. + "model": "stock.move", "archivable": False, "optional": True, + "fields": ["product_id", "quantity_done", "date", "state", + "location_id", "location_dest_id", "picking_id", "write_date"], + }, + # ⭐ THE USAGE LOOKUP, WITHOUT WHICH THE ROW ABOVE CANNOT BE READ. `location_id` syncs as + # `location_id` + `location_id_name` (the m2o convention) — a NAME, never a usage — so the + # two-sided domain has nothing to test until this table exists. It is tiny (a few hundred + # rows) and it is the whole difference between a metric and a column of doubled numbers. + # ⚠ `usage` is a selection: `internal` | `supplier` | `customer` | `inventory` | `production` + # | `transit` | `view`. ⚠ Adjustments (`inventory`) and scrap are 30% of IN / 20% of OUT and + # the two obvious exclusions are NOT equivalent — the 5,320-unit gap is all + # `Virtual Locations/Scrap` — so the metric that consumes this CHOOSES deliberately and says + # which; that choice lives in `model/metrics/stock.yml`, not here. + "stock_location": { + "model": "stock.location", "archivable": True, "optional": True, + "fields": ["name", "complete_name", "usage", "active", "write_date"], + }, +} + +BATCH = 2000 + +# Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the +# column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for +# "empty" and for "boolean false", so a bool field missing from here silently becomes NULL — and +# for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON +# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably +# from "unknown". +BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free") + +# ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence +# shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec. +# `_cols` types anything it does not recognise as VARCHAR, so `customer_rank` landed as text, the +# mirror populated correctly, every row looked right — and `odoo_relational`'s own predicate +# `customer_rank > 0` died with `Binder Error: Cannot compare values of type VARCHAR and type +# INTEGER_LITERAL`. ⛔ THE COLUMN WAS PRESENT AND THE DATA WAS CORRECT; only its TYPE was wrong, +# which is why "is the field in the mirror?" answered yes and the feature still could not run. +# ⇒ A new numeric field belongs HERE, exactly as a new boolean belongs in BOOL_FIELDS above. +INT_FIELDS = ("customer_rank", "supplier_rank") + + +import threading as _thr + +_RO_TLS = _thr.local() +_READY = {"ok": False} +_INSTANCE = {"con": None} +_INSTANCE_LOCK = _thr.Lock() +#: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with +#: the generation it was opened under, so a thread that never called `use_path` still cannot keep +#: reading the previous tenant's file — see `ro_con`. A plain "is the cursor alive?" probe cannot +#: answer that question, which is why this counter exists rather than more probing. +_GENERATION = [0] + + +def mark_ready(): + """In-process signal that the store is fully synced (every entity live). The sync writer calls + this after an all-live pass so a reader never has to (re)discover usability by touching the + file. Belt-and-suspenders alongside the shared-instance model below.""" + _READY["ok"] = True + + +def _instance(): + """THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the + sync writer uses it too — DuckDB serves concurrent reads + a single writer from ONE instance + via MVCC, so a read NEVER contends with a sync pass for the file lock. + + This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent + read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection + to same database file with a different configuration than existing connections'); a cursor on + the shared instance does not. Openers must ensure the file is fully present first — ensure_seed + writes atomically (os.replace) and every read path guards on DB_PATH.exists() — so the instance + never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole + life, so a SEPARATE process — e.g. mcp_server.py — cannot open the same store file concurrently; + in production only the app runs, and that was already true during any sync pass.)""" + con = _INSTANCE["con"] + if con is not None: + return con + with _INSTANCE_LOCK: + if _INSTANCE["con"] is None: + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + _INSTANCE["con"] = duckdb.connect(str(DB_PATH)) + return _INSTANCE["con"] + + +def ro_cursor(): + """A read cursor on the shared instance — the contention-free way to read the store.""" + return _instance().cursor() + + +def ready(): + """True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on + this: a mid-backfill store would return PARTIAL totals silently — worse than an error. Positive + results cache for the process (backfill never regresses). Reads via a shared-instance cursor, + so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight.""" + if _READY["ok"]: + return True + if not DB_PATH.exists(): + return False + try: + cur = ro_cursor() + try: + phases = {r[0]: r[1] for r in + cur.execute("SELECT entity, phase FROM _sync_state").fetchall()} + finally: + cur.close() + except Exception: + return False + # ⛔⛔ OPTIONAL ENTITIES DO NOT GATE READINESS, AND WITHOUT THIS CLAUSE ADDING ONE IS AN + # OUTAGE. `ready()` gates EVERY store read — the Analyst, every measure column, every + # read-through grid — so a new entity in `ENTITIES` makes a mirror that has never synced it + # report NOT READY and refuses all of them with "the data cache is still warming up". A Space + # hydrates from a SNAPSHOT taken before the entity existed, so this would fire on every + # deploy of a wave that adds one: a whole-product outage from a schema line, with the app + # RUNNING and every gate green. MEASURED on this box the moment `stock_move` was declared + # (W37-T13): `ready()` flipped to False against an otherwise complete mirror. + # ⚠ THE GUARANTEE IS NOT WEAKENED, IT IS SCOPED. `ready()` still means "every entity the + # EXISTING surfaces read has finished its backfill". An optional entity's own consumers must + # check ITS phase before promising a column from it — which is what + # `semantic._source_ready` does, refusing the offer with a cause rather than serving blanks. + _required = [k for k, v in ENTITIES.items() if not v.get("optional")] + if all(phases.get(k) == "live" for k in _required): + _READY["ok"] = True + return True + return False + + +def entity_live(key): + """Has THIS entity finished its backfill? The per-entity half of `ready()`. + + Exists because `ready()` is deliberately blind to optional entities (see above), so a caller + that depends on one has to ask about it specifically instead of inferring it from a global + flag that was never about them. + """ + if not DB_PATH.exists(): + return False + try: + cur = ro_cursor() + try: + row = cur.execute("SELECT phase FROM _sync_state WHERE entity = ?", [key]).fetchone() + finally: + cur.close() + except Exception: # noqa: BLE001 + return False + return bool(row) and row[0] == "live" + + +def ro_con(): + """Per-thread cached read CURSOR on the shared instance for page/module store reads (the + open cost dominates repeated small queries — the Expenses retrofit measured 7.7s→1.0s). No + lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES + until ready() — callers either fall back to live reads (modules) or relay a readable message + (Analyst). + + ⚠ THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness + probe below asks "does this cursor still work?", which is a different question from "is this + cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches + tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` — + and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no + exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is + discarded here, which is the only place every thread is guaranteed to pass through.""" + cur = getattr(_RO_TLS, "cur", None) + if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]: + try: + cur.close() + except Exception: + pass + cur, _RO_TLS.cur = None, None # the store moved underneath this thread + if cur is not None: + try: + cur.execute("SELECT 1") # cheap liveness probe + return cur + except Exception: + _RO_TLS.cur = None # stale → reopen below + if not ready(): + raise RuntimeError("the tenant data store is completing its FIRST sync on this " + "deployment — dashboards work meanwhile (live reads); retry " + "store queries in a few minutes") + cur = ro_cursor() + _RO_TLS.cur = cur + _RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to + return cur + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# ⭐ WAVE 30 / OWNER RULING R6 + R7 — THE WINDOW. Read this before touching `window()`. +# +# R6, verbatim: *"there is no cap in how many data from the API source (as long as its from a +# connected source like Odoo) that can be pulled into the app."* R7 says how: a connected grid +# reads THROUGH this mirror instead of copying rows into the one `user_tables` document, which +# `MAX_ROWS = 60_000` bounds. The mirror is already uncapped — 971,034 GL lines live in this file +# today — so "no cap" is not a bigger number, it is a different mechanism: serve a SLICE and count +# the whole. +# +# ⛔⛔ `total` IS A `SELECT count(*)` OVER THE SAME PREDICATE, NEVER `len(rows)`. A window whose +# count is its own length is a fabricated aggregate that reads as authoritative — the class this +# repo has paid for twice ([[no-unverifiable-aggregates]], [[one-question-two-normalizers]]: a +# display predicate and a fold predicate answering one question differently). The two statements +# below are built from ONE `where` + ONE `params` tuple for exactly that reason; they cannot drift +# apart without deleting a line. +# +# ⛔ AND THE PREDICATE PUSHES DOWN. `where` is compiled by `harness/filter_sql.py` — the same +# compiler the client's filter engine is held in step with — so a filter matching rows outside the +# loaded window still COUNTS them. A caller that filters the returned list instead has silently +# asked "how many of the 200 rows in memory match" about a 971,034-row table. +# +# ⚠ `order_by` IS REQUIRED IN PRACTICE AND DEFAULTED HERE. Two pages of an UNORDERED window are +# not guaranteed to partition the table: DuckDB may legally return a row on page 1 and again on +# page 2, and the user sees a duplicate with no error anywhere. The default is the physical +# rowid-ish `1` only when a caller genuinely has no key; every real caller passes one. +# +# ⚠ TRUST BOUNDARY. `table`, `select`, `where` and `order_by` are SQL we author (a spec row, or +# `filter_sql`'s output over a whitelisted column map). Every VALUE is bound through `params`. +# `table` is additionally shape-checked below — not because a caller is hostile, but because a +# typo'd identifier interpolated into two statements is worth one cheap assertion. + +#: The most rows ONE request may carry back. Not a cap on the data — every row is reachable by +#: paging and `total` always tells the truth about how many there are — but a bound on the memory +#: a single response can cost. R6's second sentence applies: a caller that asks for more is CLAMPED +#: and the clamp is REPORTED (`routes_odoo_tables` turns it into a `limits` entry), never silent. +WINDOW_MAX = 5_000 + +#: What a caller gets when it names no window at all. Matches `semantic.store_rows`' own default so +#: the two windowed readers in this codebase do not disagree about what "a page" means. +WINDOW_DEFAULT = 200 + +_IDENT_OK = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def window(table=None, select=None, where=None, params=(), order_by=None, + offset=0, limit=WINDOW_DEFAULT, cur=None, from_sql=None): + """One page of a mirror table PLUS the true count of everything the same predicate matches. + + Returns `{"rows": [tuple, ...], "columns": [name, ...], "total": int, "offset": int, + "limit": int, "clamped": bool}` — `rows` is the slice, `total` is the population. + + `select` is the projection expression list ("id, name, amount_untaxed"); `where` is a WHERE + body with `?` placeholders; `params` binds them, and is used by BOTH statements. + + ⚠ `table` and `from_sql` are the SAME argument wearing two trust levels, and they are separate + names on purpose. `table` is an identifier and is SHAPE-CHECKED; `from_sql` is a whole FROM + body — a parenthesised subquery with an alias — which cannot be checked at all. Two of the + eight connected grids need one (`read_customers` and `read_agents` are single statements whose + population is a SQL `UNION`), so the capability has to exist; naming it distinctly means a + reviewer can grep for every caller that hands over raw FROM SQL instead of inferring it. + """ + if bool(table) == bool(from_sql): + raise ValueError("datastore.window: pass exactly one of `table` or `from_sql`") + if from_sql: + name = str(from_sql).strip() + else: + name = str(table or "").strip() + if not _IDENT_OK.match(name): + raise ValueError(f"datastore.window: {table!r} is not a table identifier") + proj = str(select or "").strip() + if not proj: + raise ValueError("datastore.window: a projection is required — there is no implicit *") + try: + offset = max(0, int(offset or 0)) + except (TypeError, ValueError): + offset = 0 + try: + limit = int(limit if limit is not None else WINDOW_DEFAULT) + except (TypeError, ValueError): + limit = WINDOW_DEFAULT + clamped = limit > WINDOW_MAX or limit < 1 + limit = min(max(limit, 1), WINDOW_MAX) + + params = tuple(params or ()) + pred = str(where or "").strip() + tail = f" WHERE {pred}" if pred else "" + order = str(order_by or "").strip() or "1" + cur = cur if cur is not None else ro_con() + + # ⛔ THE COUNT RUNS FIRST AND OVER THE SAME `tail` + `params`. Ordering it first is deliberate: + # if the projection is ever wrong, the caller fails LOUDLY on the rows query rather than + # quietly returning a good count beside a broken page. + total = int(cur.execute(f"SELECT count(*) FROM {name}{tail}", params).fetchone()[0] or 0) + got = cur.execute( + f"SELECT {proj} FROM {name}{tail} ORDER BY {order} LIMIT {int(limit)} OFFSET {int(offset)}", + params).fetchall() + cols = [d[0] for d in (cur.description or [])] + return {"rows": [tuple(r) for r in got], "columns": cols, + "total": total, "offset": offset, "limit": limit, "clamped": clamped} + + +def columns_of(table, cur=None): + """The column names a mirror table actually has, lowercased. + + ⚠ A mirror can be `ready()` and still be MISSING COLUMNS: `ready()` reads entity PHASES, while + `_ensure_columns`/`_backfill_columns` checkpoint separately. Asking before projecting is what + stops a DuckDB `BinderException: Referenced column … not found` reaching a user as a bare 500 + (it already cost one live, which is why `odoo_relational.columns` exists on the app side). + """ + name = str(table or "").strip() + if not _IDENT_OK.match(name): + raise ValueError(f"datastore.columns_of: {table!r} is not a table identifier") + cur = cur if cur is not None else ro_con() + try: + rows = cur.execute(f"PRAGMA table_info('{name}')").fetchall() + except Exception: # noqa: BLE001 + return set() + return {str(r[1]).lower() for r in rows} + + +SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data") + + +def ensure_seed(): + """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store + from the PRIVATE dataset's seed snapshot — Analyst-ready in ~a minute instead of a 20-40 min + XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores + the pilot views/routines when absent. Fail-quiet: no seed/token → normal backfill.""" + import shutil + if DB_PATH.exists(): + return False + tok = os.environ.get("HF_TOKEN") + if not tok: + return False + try: + from huggingface_hub import hf_hub_download + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb", + repo_type="dataset", token=tok) + # Atomic install: copy to a temp path then os.replace — a reader that guards on + # DB_PATH.exists() must never bind a half-written file into the shared instance. + tmp = DB_PATH.with_name(DB_PATH.name + ".tmp") + shutil.copyfile(p, tmp) + os.replace(tmp, DB_PATH) + for fn in ("views.json", "routines.json"): + tgt = DB_PATH.parent / fn + if not tgt.exists(): + try: + q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}", + repo_type="dataset", token=tok) + shutil.copyfile(q, tgt) + except Exception: + pass + return True + except Exception: + return False + + +def connect(): + """A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a + cursor so callers' con.close() frees the cursor without closing the process-wide instance; + one writer cursor is active at a time (the sync thread is sequential), readers use others.""" + cur = _instance().cursor() + cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state ( + entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, + cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""") + return cur + + +def _flatten(rec, fields, m2m=()): + out = {"id": rec["id"]} + for f in m2m: + v = rec.get(f) or [] + base = f.removesuffix("_ids") + out[f"{base}_id"] = v[0] if v else None + out[f] = json.dumps(list(v)) if v else None + for f in fields: + v = rec.get(f) + if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int): + out[f"{f.removesuffix('_id')}_id"] = v[0] + out[f"{f.removesuffix('_id')}_name"] = str(v[1]) + elif isinstance(v, (list, tuple)): + out[f] = json.dumps(v) + elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields + out[f] = None + else: + out[f] = v + return out + + +#: ⛔⛔ A HARD-CODED NAME LIST DECIDES WHETHER A COLUMN IS A NUMBER, and a field missing from it +#: lands as VARCHAR — which does not fail, it fails LATER and somewhere else. MEASURED (W37-T13): +#: `stock.move.quantity_done` was absent, so 778,385 rows stored '4.0', '1.0' … as TEXT, every one +#: of them cleanly castable, and the only symptom was a DuckDB +#: *"Cannot mix values of type INTEGER_LITERAL and VARCHAR in CASE expression"* thrown by a +#: FILTERED MEASURE three layers up in `semantic._measure_sql`. Nothing in the sync, the schema or +#: the row count looked wrong. +#: ⚠ SO ADDING A NUMERIC FIELD TO AN `ENTITIES` SPEC MEANS ADDING IT HERE TOO. There is no +#: inference: `_row` writes whatever Odoo returned, and Odoo returns `False` for an unset numeric, +#: so a value-sniffing default would be wrong in the other direction. +#: ⚠ AND THE TYPE IS FIXED AT CREATE. `_ensure_columns` only ADDs missing columns — it never ALTERs +#: an existing one — so a mirror that already synced the field as VARCHAR keeps it until the table +#: is dropped and re-synced. Correcting the list fixes every FRESH mirror, which is the Space's +#: case (it has never synced this entity) and was not mine (I had to ALTER my scratch copy). +NUMERIC_FIELDS = ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed", + "price_subtotal", "margin", "purchase_price", "product_uom_qty", + "standard_price", "debit", "credit", "balance", "amount", + # W37-T13 — the stock-movement quantity. See the note above. + "quantity_done") + + +def _cols(spec): + cols = ["id BIGINT PRIMARY KEY"] + for f in spec["fields"]: + base = f.removesuffix("_id") + if f.endswith("_id"): + cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"] + elif f in BOOL_FIELDS: + cols.append(f"{f} BOOLEAN") + elif f in INT_FIELDS: + cols.append(f"{f} BIGINT") + elif f in NUMERIC_FIELDS: + cols.append(f"{f} DOUBLE") + else: + cols.append(f"{f} VARCHAR") + for f in spec.get("m2m") or []: + base = f.removesuffix("_ids") + cols += [f"{base}_id BIGINT", f"{f} VARCHAR"] + return cols + + +def _ensure_table(con, key, spec): + con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})") + + +def _ensure_columns(con, key, spec): + """Schema evolution: existing stores/seeds predate columns a newer spec introduces — + CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names.""" + have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()} + added = [] + for c in _cols(spec): + name = c.split()[0] + if name not in have: + con.execute(f"ALTER TABLE {key} ADD COLUMN {c}") + added.append(name) + return added + + +def _field_cols(f): + """The store column(s) one spec field materialises into (m2o fields become _id + _name).""" + if f.endswith("_id"): + base = f.removesuffix("_id") + return [f"{base}_id", f"{base}_name"] + return [f] + + +def _backfill_columns(con, key, spec, log=print): + """Scalar/m2o schema evolution — the twin of _backfill_m2m, and for the same reason. + + _ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema; + their write_date didn't move), so a newly added field stays NULL forever — silently, and NULL + is indistinguishable from a legitimately empty value. That is how a store starts answering + 'no rows match' instead of erroring. Re-pull just the affected fields for every row, + paginated by id, marked durably so a crash retries rather than declaring completion. + + The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns + did _ensure_columns just add" — that inference is wrong on the second run, when the ALTER has + already happened and the list comes back empty while the data is still missing. Declaring it + makes the migration re-entrant and reviewable. Leave the entry in place after it completes; + the durable marker, not the absence of the declaration, is what stops it re-running. + """ + fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"] + if not fields: + return + marker = f"{key}.cols.{','.join(sorted(fields))}" + st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?", + [marker]).fetchone() + if st and st[0] == "done": + return + # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses + # an hour of work to one dropped connection gets skipped by the next person under time + # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an + # interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring + # completion it never reached. + last, n = (st[1] or 0, st[2] or 0) if st else (0, 0) + if last: + log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)") + cols = [c for f in fields for c in _field_cols(f)] + setter = ", ".join(f"{c}=?" for c in cols) + while True: + recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], + fields, limit=5000, order="id asc") + if not recs: + break + rows = [_flatten(r, fields) for r in recs] + con.executemany(f"UPDATE {key} SET {setter} WHERE id=?", + [[r.get(c) for c in cols] + [r["id"]] for r in rows]) + last, n = recs[-1]["id"], n + len(recs) + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)") + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: column backfill complete -> {n:,} rows") + + +def _backfill_m2m(con, key, spec, log=print): + """One-time targeted backfill after a schema migration: existing rows never re-sync (their + write_date didn't move when WE added a column), so pull every record where the m2m field is + SET and update in place. Rows with the field empty keep NULL — correct and free. Completion + is a durable _sync_state marker ('.m2m.') — NOT 'column just added': a crash + between the ALTER and this backfill (seen live: the SSL-failed first run) must retry.""" + for f in spec.get("m2m") or []: + marker = f"{key}.m2m.{f}" + if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone(): + continue + base = f.removesuffix("_ids") + # PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line + # has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone + # missing with no error — a partial backfill that reads as a complete one (the + # [[no-unverifiable-aggregates]] no-silent-caps rule). + rows, last, page = [], 0, 20000 + while True: + recs = O.search_read(spec["model"], + _base_domain(spec) + [(f, "!=", False), ("id", ">", last)], + [f], limit=page, order="id asc") + if not recs: + break + rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)] + last = recs[-1]["id"] + log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)") + if rows: + con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows) + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + [marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")]) + log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated") + + +def _upsert(con, key, spec, recs): + if not recs: + return + rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs] + colnames = [c.split()[0] for c in _cols(spec)] + ids = [r["id"] for r in rows] + con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})") + con.executemany( + f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})", + [[r.get(c) for c in colnames] for r in rows]) + + +def _state(con, key): + row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?", + [key]).fetchone() + return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None + + +def _save_state(con, key, phase, last_id, cursor_wd): + n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""", + [key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")]) + + +def _base_domain(spec): + return [("active", "in", [True, False])] if spec["archivable"] else [] + + +def sync_entity(key, max_batches=200, batch=BATCH, log=print): + """One bounded, resumable sync pass for an entity. Backfill (by id) → live (by write_date). + Returns (phase, pulled_rows). + + ⭐ D-10: refuses BEFORE opening a cursor or touching Odoo while the connector is paused. The + guard is here — the innermost function that reads from `O` — rather than only in `sync_all`, + so a caller that syncs ONE entity is covered by construction instead of by remembering to + add the same check. `sync_all` short-circuits too, but only to avoid eight identical notices. + """ + if source_paused(): + _paused_notice(f"sync of {key}", log) + return PAUSED_PHASE, 0 + spec = ENTITIES[key] + con = connect() + _ensure_table(con, key, spec) + _ensure_columns(con, key, spec) + _backfill_columns(con, key, spec, log=log) + _backfill_m2m(con, key, spec, log=log) + st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0} + fields = spec["fields"] + list(spec.get("m2m") or []) + pulled = 0 + try: + if st["phase"] == "backfill": + last_id = st["last_id"] or 0 + for i in range(max_batches): + dom = _base_domain(spec) + [("id", ">", last_id)] + recs = O.search_read(spec["model"], dom, ["id"] + fields, + limit=batch, order="id asc") + if not recs: + # backfill complete → switch to live mode anchored at max write_date seen + wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0] + _save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00") + log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode") + return "live", pulled + _upsert(con, key, spec, recs) + last_id = recs[-1]["id"] + pulled += len(recs) + _save_state(con, key, "backfill", last_id, None) + if (i + 1) % 10 == 0: + log(f" {key}: backfill …{pulled:,} rows (id≤{last_id})") + log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)") + return "backfill", pulled + # live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP + # FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE + # write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15 + # 16:35:03'; stored values carry microseconds, XML-RPC strings don't) — then the + # write_date cursor can NEVER advance and every pass re-pulls the same first page + # forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over + # write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|' in + # cursor_wd so a killed walk resumes; on completion the cursor jumps one second past + # the stamp (safe: the walk covered every row in that second, and the stamp is in the + # past — guarded by a 2-minute recency check). + raw = st["cursor_wd"] or "1970-01-01 00:00:00" + cursor, _pipe, _tie = raw.partition("|") + walking, tie_id = bool(_pipe), int(_tie or 0) + walk_max = cursor + for _ in range(max_batches): + if walking: + dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)] + recs = O.search_read(spec["model"], dom, ["id"] + fields, + limit=batch, order="id asc") + if not recs: # walk complete → advance past the stamp + try: + wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S") + if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2): + cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S") + else: + cursor = walk_max + except ValueError: + cursor = walk_max + walking = False + log(f" {key}: id-walk complete -> cursor {cursor}") + break + _upsert(con, key, spec, recs) + pulled += len(recs) + tie_id = recs[-1]["id"] + walk_max = max(walk_max, max(str(r["write_date"]) for r in recs)) + _save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}") + continue + dom = _base_domain(spec) + [("write_date", ">=", cursor)] + recs = O.search_read(spec["model"], dom, ["id"] + fields, + limit=batch, order="write_date asc, id asc") + if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor + and pulled == 0 and _already(con, key, recs[0])): + break + new_cursor = str(recs[-1]["write_date"]) + if new_cursor == cursor and len(recs) == batch: + # full batch stuck on one second: enter the id-walk from id 0 (the write_date + # ordering shuffles ids within the stamp second, so the batch just pulled is + # NOT an id prefix — restart coverage by id; upserts make the overlap free) + walking, tie_id, walk_max = True, 0, cursor + _save_state(con, key, "live", st["last_id"], f"{cursor}|0") + log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk") + continue + _upsert(con, key, spec, recs) + pulled += len(recs) + if new_cursor == cursor and len(recs) < batch: + cursor = new_cursor + break + cursor = new_cursor + # Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the + # 2026-07-15 recompute) means a killed pass otherwise re-pulls everything — + # upserts are idempotent, but the wasted RPC is real (learned 2026-07-17). + _save_state(con, key, "live", st["last_id"], cursor) + _save_state(con, key, "live", st["last_id"], + f"{cursor}|{tie_id}" if walking else cursor) + if pulled: + log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}" + + (f" walking id>{tie_id}" if walking else "") + ")") + return "live", pulled + finally: + con.close() + + +def _already(con, key, rec): + return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone()) + + +def sync_all(entities=None, max_batches=200, log=print): + # ⭐ D-10. `{}` rather than a dict of paused phases, and the caller decides why that matters: + # `api/main.py:354` loops up to 12 times until `all(phase == "live")`, and `all()` over an + # empty dict is True — so a paused store costs ONE pass instead of twelve rounds of the same + # refusal. A dict of `"paused"` phases would fail that test every time and turn the boot + # sprint into a busy-wait against a connector nobody has resumed. + if source_paused(): + _paused_notice("sync", log) + return {} + out = {} + for key in (entities or list(ENTITIES)): + t0 = time.time() + phase, pulled = sync_entity(key, max_batches=max_batches, log=log) + out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)} + # A full pass that lands every entity in live mode latches process readiness in-process — so + # readers never race the writer's lock to (re)discover the store is usable. + if entities is None and out and all(v["phase"] == "live" for v in out.values()): + mark_ready() + return out + + +def reconcile_deletes(entities=None, log=print): + """Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor + sync can never see a deletion — the count parity DETECTS the drift (by design); this repairs + it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are + cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the + app's frequent passes. + + ⭐ D-10: it pulls the FULL live id set per entity, so it is the single largest Odoo read in + this module and it runs unattended at boot and every fourth resync pass. Paused means paused. + """ + if source_paused(): + _paused_notice("delete reconcile", log) + return {} + con = connect() + try: + out = {} + for key in (entities or list(ENTITIES)): + spec = ENTITIES[key] + live_ids, last = set(), 0 + while True: + recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)], + ["id"], limit=50000, order="id asc") + if not recs: + break + live_ids.update(r["id"] for r in recs) + last = recs[-1]["id"] + store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()} + dead = store_ids - live_ids + if dead: + con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)") + con.execute("DELETE FROM _dead") + con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead]) + con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)") + con.execute("DELETE FROM _dead") + n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key]) + out[key] = len(dead) + log(f" {key}: reconciled {len(dead):,} hard-deleted rows") + return out + finally: + con.close() + + +def status(): + con = connect() + try: + return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]} + for r in con.execute("SELECT * FROM _sync_state").fetchall()} + finally: + con.close() + + +# ------------------------------------------------------------------ parity (store vs LIVE Odoo) + +def _check(name, ours, live, tol=0.01): + gap = abs((ours or 0) - (live or 0)) + return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live} + + +def validate(pre=None, months=("2026-01-01", "2026-07-01")): + """Raw-fidelity parity: the store must mirror live Odoo — row counts per synced entity + + unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase. + + ⭐ D-10 REFUSES LOUDLY HERE rather than skipping quietly, and the difference matters. Every + other paused path has a correct silent answer — serve the mirror — because the mirror is the + thing being asked for. This function's ENTIRE contract is "compare me against live Odoo", so + a paused version of it has no honest result: skipping would return an empty pass, and + proceeding would make the live call the pause forbids. An operator running a reconciliation + against a paused connector has asked a question that cannot be answered, and being told so is + the only outcome that is not a lie. (Operator-invoked only — nothing in the serving path + calls it, so this raises for a person, never inside a request.) + """ + if source_paused(): + raise RuntimeError( + "the tenant's Odoo connector is PAUSED, so the store cannot be validated against " + "live Odoo - resume it under Settings > Connectors and run this again") + con = connect() + out = [] + try: + live_state = status() + d0, d1 = months + for key, spec in ENTITIES.items(): + if live_state.get(key, {}).get("phase") != "live": + continue + ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0] + live = O.get_odoo().search_count(spec["model"], _base_domain(spec)) + out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0)) + if live_state.get("sale_order_line", {}).get("phase") == "live": + ours = con.execute( + "SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l " + "JOIN sale_order o ON o.id = l.order_id " + "WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0] + live = O.sum_field("sale.order.line", + [("order_id.date_order", ">=", f"{d0} 00:00:00"), + ("order_id.date_order", "<", f"{d1} 00:00:00")], + "price_subtotal") + out.append(_check(f"store Σ line revenue [{d0}..{d1}) == live (unscoped)", ours, live)) + if live_state.get("sale_order", {}).get("phase") == "live": + ours = con.execute( + "SELECT coalesce(sum(amount_untaxed),0) FROM sale_order " + "WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0] + live = O.sum_field("sale.order", + [("date_order", ">=", f"{d0} 00:00:00"), + ("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed") + out.append(_check(f"store Σ order amount_untaxed [{d0}..{d1}) == live (unscoped)", + ours, live)) + finally: + con.close() + return out + + +if __name__ == "__main__": + args = sys.argv[1:] + if args and args[0] == "status": + for k, v in status().items(): + print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})") + elif args and args[0] == "reconcile": + ents = args[1].split(",") if len(args) > 1 else None + for k, n in reconcile_deletes(ents).items(): + print(f" {k:18s} -{n:,} hard-deleted rows") + elif args and args[0] == "validate": + ok = True + for c in validate(): + mark = "OK " if c["ok"] else "XX " + ok &= c["ok"] + print(f" {mark}{c['check'][:80]} gap={c['gap']}") + print("PARITY:", "GREEN" if ok else "RED") + sys.exit(0 if ok else 1) + else: + ents = args[0].split(",") if args else None + mb = int(args[1]) if len(args) > 1 else 200 + for k, v in sync_all(ents, max_batches=mb).items(): + print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s") diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index 1d65de835b6f4d5b1c9a038a0141d3e28ed63b90..4a87dce4ac0c3ac1cd355bf24b471e9ffcf2d1c1 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -1,974 +1,1502 @@ -"""harness/semantic.py — the semantic seam (OM-0a, 2026-07-11). - -The ONE governed registry of topics + metrics (platform/model/*.yml) and the resolver that -turns a metric key into a number through the SAME proven data layer the validated modules use -(core/odoo domain builders + aggregate helpers) — so scope parity is by construction, and every -metric carries its definition, ai_context, and an independent validate contract. - -Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved- -dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4: -list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan: -.claude/wiki/research/omni-adoption.md (Part IV + the productization addendum). - -Design constraints honored (from /odoo-api): - - read_group dot-path GROUPBY fails → scalar aggregates here use sum_field/distinct_count; - grouped queries (OM-3+) must group by direct m2o fields only. - - Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...); - the YAML `scope:` block is the declarative statement of what that builder bakes in. - - READ-ONLY on Odoo, always. -""" -import ast -import functools -import operator -from pathlib import Path - -import yaml - -import core.odoo as O - -MODEL_DIR = Path(__file__).resolve().parents[1] / "model" - -# Topic scope binds to a NAMED, reviewed domain builder — never a YAML-assembled domain (one -# source of truth for scope until OM-2 makes topics executable against the store). -def _order_domain(*a, **kw): - # lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store - import modules.sales as S - return S.order_domain(*a, **kw) - - -def _customer_move_domain(move_type): - """A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly. - - ⚠ COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry - team_id = 1, so a BU filter on the document is meaningless — silently dropping the argument - would hand a business-unit user the company's number under their own name, which is the - widening sin in a different currency. - """ - def build(date_from=None, date_to=None, team_id=None, **kw): - if team_id: - raise ModelError( - f"customer {move_type} documents are COMPANY-LEVEL — credit notes and invoices " - f"carry no trustworthy business-unit tag, so a team-scoped total cannot be " - f"produced. Refusing rather than returning the company number.") - dom = [("move_type", "=", move_type), ("state", "=", "posted")] - if date_from: - dom.append(("invoice_date", ">=", date_from)) - if date_to: - dom.append(("invoice_date", "<=", date_to)) - return dom - return build - - -_BUILDERS = { - "sale_line_domain": O.sale_line_domain, - "sale_order_domain": _order_domain, - "credit_note_domain": _customer_move_domain("out_refund"), - "customer_invoice_domain": _customer_move_domain("out_invoice"), -} - - -class ModelError(Exception): - """A model-file problem (unknown key, bad reference, unparseable expr).""" - - -# ---------------------------------------------------------------- loading - -@functools.lru_cache(maxsize=1) -def _tenant(): - f = MODEL_DIR / "tenant.yml" - return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}} - - -def _sql_value(v): - """Render a tenant param into SQL safely (params are versioned config, not user input — - but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped.""" - if isinstance(v, bool): - raise ModelError("boolean tenant params not supported in scope_sql") - if isinstance(v, (int, float)): - return str(v) - if isinstance(v, (list, tuple)): - return ", ".join(_sql_value(x) for x in v) - return "'" + str(v).replace("'", "''") + "'" - - -def render_scope(sql): - """Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml.""" - params = _tenant().get("params") or {} - out = sql - for k, v in params.items(): - out = out.replace("{" + k + "}", _sql_value(v)) - if "{" in out and "}" in out: - import re as _re - missing = _re.findall(r"\{([a-z_]+)\}", out) - if missing: - raise ModelError(f"scope_sql references unknown tenant params: {missing}") - return out - - -@functools.lru_cache(maxsize=1) -def _model(): - topics, metrics = {}, {} - for f in sorted((MODEL_DIR / "topics").glob("*.yml")): - d = yaml.safe_load(f.read_text(encoding="utf-8")) - if not d or "key" not in d: - raise ModelError(f"topic file {f.name} lacks a 'key'") - if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS: - raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}") - # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises - # a clear error if a live resolution is attempted on it - topics[d["key"]] = d - for f in sorted((MODEL_DIR / "metrics").glob("*.yml")): - d = yaml.safe_load(f.read_text(encoding="utf-8")) - topic = d.get("topic") - if topic not in topics: - raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}") - for m in d.get("metrics", []): - if "key" not in m: - raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'") - if m["key"] in metrics: - raise ModelError(f"duplicate metric key {m['key']!r}") - m.setdefault("topic", topic) # a metric may name its own topic (rare) - if m["topic"] not in topics: - raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}") - metrics[m["key"]] = m - return {"topics": topics, "metrics": metrics} - - -def reload_model(): - _model.cache_clear() - _tenant.cache_clear() - return _model() - - -def topics(): - return dict(_model()["topics"]) - - -def metrics(): - return dict(_model()["metrics"]) - - -def describe(key): - """Full metadata for a metric (the dictionary page / AI describe_topic feed).""" - m = _model()["metrics"].get(key) - if not m: - raise ModelError(f"unknown metric {key!r}") - return {**m, "topic_def": _model()["topics"][m["topic"]]} - - -# ---------------------------------------------------------------- resolving - -def _domain(topic_def, date_from, date_to, team_id): - b = topic_def.get("domain_builder") - if not b: - raise ModelError(f"topic {topic_def['key']!r} is store-only — use store_query() " - f"(no live-path domain builder bound yet)") - return _BUILDERS[b](date_from, date_to, team_id=team_id) - - -_OPS = {ast.Add: operator.add, ast.Sub: operator.sub, - ast.Mult: operator.mul, ast.Div: operator.truediv} - - -def _eval_expr(expr, resolve): - """Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens.""" - def ev(node): - if isinstance(node, ast.Expression): - return ev(node.body) - if isinstance(node, ast.BinOp) and type(node.op) in _OPS: - return _OPS[type(node.op)](ev(node.left), ev(node.right)) - if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): - return -ev(node.operand) - if isinstance(node, ast.Num): # py<3.8 compat name; Constant below - return node.n - if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): - return node.value - if isinstance(node, ast.Name): - return resolve(node.id) - raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}") - try: - return ev(ast.parse(expr, mode="eval")) - except SyntaxError as e: - raise ModelError(f"unparseable derived expr {expr!r}: {e}") - - -def metric(key, date_from=None, date_to=None, team_id=None, _seen=None): - """Resolve a registered metric to a number for a window (+optional BU). The only public - compute path — every surface that shows this metric goes through here.""" - _seen = _seen or set() - if key in _seen: - raise ModelError(f"circular metric reference at {key!r}") - _seen = _seen | {key} - m = _model()["metrics"].get(key) - if not m: - raise ModelError(f"unknown metric {key!r}") - agg = m.get("agg") - if agg == "ratio": - num = metric(m["numerator"], date_from, date_to, team_id, _seen) - den = metric(m["denominator"], date_from, date_to, team_id, _seen) - return (num / den) if den else 0.0 - if agg == "derived": - return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen)) - t = _model()["topics"][m["topic"]] - dom = _domain(t, date_from, date_to, team_id) - # Wave 21 R2 — the LIVE twin of `store_filter_sql`. A filtered metric must filter BOTH - # paths, or store_parity would compare a narrowed store number against an unfiltered live - # one and the "parity failure" would read as a data problem. YAML shape: - # live_domain: [[field, op, value], ...] — appended verbatim to the topic domain. - for c in (m.get("live_domain") or []): - dom = dom + [tuple(c)] - if agg == "sum": - # ⚠ `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a - # STORE-ONLY topic, so the live path never met it — the first metric that is both negated - # and parity-checked (`returns`) would otherwise compare a negative live number against a - # positive store one, and the "parity failure" would look like a data problem. - got = O.sum_field(t["entity"], dom, m["field"]) - return -got if m.get("negate") else got - if agg == "count_distinct": - return O.distinct_count(t["entity"], dom, m["field"]) - if agg == "count": - return O.get_odoo().search_count(t["entity"], dom) - raise ModelError(f"metric {key!r}: unknown agg {agg!r}") - - -# ---------------------------------------------------------------- validate contracts - -def _v_order_level_revenue(date_from, date_to, team_id): - """Independent check: line-level Σ price_subtotal == order-level Σ amount_untaxed.""" - import modules.sales as S - line = metric("revenue", date_from, date_to, team_id) - order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed") - return line, order - - -def _v_order_count(date_from, date_to, team_id): - """Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders. - (Confirmed orders with zero lines exist in the data — 2 found at first proof, 2026-07-11; - counting them explicitly makes the identity exact instead of hiding them in a tolerance, - and surfaces them as a data-health signal.)""" - import modules.sales as S - order_level = metric("orders", date_from, date_to, team_id) - dom = S.order_domain(date_from, date_to, team_id) - from_lines = O.distinct_count("sale.order.line", - O.sale_line_domain(date_from, date_to, team_id), "order_id") - empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)]) - return order_level, from_lines + empty - - -def _v_gl_opex(date_from, date_to, team_id): - """Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted, - expense-type accounts), to the cent. team_id N/A (company-level).""" - res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to) - ours = res["rows"][0]["opex"] if res["rows"] else 0 - dom = [("move_id.state", "=", "posted"), - ("account_id.account_type", "in", ["expense", "expense_depreciation"])] - if date_from: - dom.append(("date", ">=", date_from)) - if date_to: - dom.append(("date", "<=", date_to)) - live = O.sum_field("account.move.line", dom, "balance") - return ours, live - - -def _v_ar_outstanding(date_from, date_to, team_id): - """Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals). - No date window — outstanding is an as-of-now balance.""" - res = store_query("receivables", ["ar_outstanding"]) - ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0 - live = O.sum_field("account.move", - [("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])], - "amount_residual_signed") - return ours, live - - -def _v_returns_sign(date_from, date_to, team_id): - """Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the - same posted credit notes, read straight from Odoo without going through the metric. - - A lost negation makes returns negative; a doubled one makes it negative again. Both read as - "no returns" rather than as an error, and neither is visible to any self-consistency check — - the store and the live path would agree with each other perfectly while both being wrong. - """ - ours = metric("returns", date_from, date_to, team_id) - raw = O.sum_field("account.move", - _BUILDERS["credit_note_domain"](date_from, date_to, team_id), - "amount_untaxed_signed") - return ours, abs(raw) - - -def _v_line_vs_move_grain(date_from, date_to, team_id): - """Independent check on the invoice-LINE topic: the store's netted product-line total must - equal the same figure read straight from LIVE Odoo. - - Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION — - `price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates - revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER — the - invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including - them changes the total silently rather than erroring. - - Company-level (team_id None) uses the strongest possible reference: the MOVE-grain - `amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue - (account.move carries no business unit — every invoice sits on team 1), so the reference is - the live line-level aggregate reached through the sale-order dot path: still an independent - engine and query path from the DuckDB store, which is what the contract requires. - """ - ours = (store_query("invoice_lines", ["invoiced_line_sales"], - date_from=date_from, date_to=date_to, team_id=team_id) - .get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0 - if team_id is None: - raw = O.sum_field("account.move", - [("state", "=", "posted"), - ("move_type", "in", ["out_invoice", "out_refund"]), - ("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)], - "amount_untaxed_signed") - return ours, raw - dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"), - ("date", ">=", date_from), ("date", "<=", date_to), - ("sale_line_ids.order_id.team_id", "=", team_id)] - gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")], - "price_subtotal") - refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")], - "price_subtotal") - return ours, (gross or 0) - (refunds or 0) - - -_VALIDATORS = { - "returns_sign": _v_returns_sign, - "line_vs_move_grain": _v_line_vs_move_grain, - "order_level_revenue": _v_order_level_revenue, - "order_count": _v_order_count, - "gl_opex": _v_gl_opex, - "ar_outstanding": _v_ar_outstanding, -} - - -def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01): - """Run a metric's declared independent cross-check. Returns the standard check dict - (the validate.py convention: check/ok/gap).""" - m = _model()["metrics"].get(key) - if not m: - raise ModelError(f"unknown metric {key!r}") - contract = m.get("validate") - if not contract: - return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0, - "note": "declare one in model/metrics when an independent aggregate exists"} - fn = _VALIDATORS.get(contract["method"]) - if not fn: - raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}") - ours, independent = fn(date_from, date_to, team_id) - gap = abs((ours or 0) - (independent or 0)) - return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})", - "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent} - - -def validate(pre=None): - """Module-convention validate(): run every declared contract over YTD, all-BU + per-BU.""" - import core.periods as P - yf, yt = P.ytd() - out = [] - for key, m in _model()["metrics"].items(): - if not m.get("validate"): - continue - out.append(validate_metric(key, yf, yt, None)) - topic = _model()["topics"][m["topic"]] - if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs - for tid in O.TEAM_IDS: - out.append(validate_metric(key, yf, yt, tid)) - return out - - -# ---------------------------------------------------------------- store path (OM-2) -# The compiler behind run_semantic_query: registered metrics + whitelisted dims → DuckDB SQL over -# the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files -# (trusted); every VALUE is parameterized — the small model only ever picks keys, so there is no -# injection surface. Live XML-RPC stays the validate() path (the store never validates itself). - -def _store_con(): - import harness.datastore as DS - if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently - raise ModelError("the data cache is still warming up after a restart (a one-time first " - "sync) — the dashboards work now from live data; ask me again in a " - "moment and I'll have it") - return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention - - -def _measure_sql(m, alias): - """SQL for a base measure. Supports FILTERED measures (`store_filter_sql` — the Omni - filtered-measure concept: sum(CASE WHEN … )) and `negate: true` (e.g. GL income, stored - credit-negative, reported positive).""" - agg, f = m.get("agg"), m.get("field") - flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None - if agg == "sum": - core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})" - elif agg == "count_distinct": - core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt - else f"count(DISTINCT {alias}.{f})") - elif agg == "count": - core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)" - else: - raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression") - return f"-({core})" if m.get("negate") else core - - -def _expand_measures(keys): - """Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the - requested keys for post-compute. Returns (base_keys, requested_keys).""" - mts = _model()["metrics"] - base, seen = [], set() - - def need(k): - if k in seen: - return - seen.add(k) - m = mts.get(k) - if not m: - raise ModelError(f"unknown metric {k!r}") - if m["agg"] == "ratio": - need(m["numerator"]); need(m["denominator"]) - elif m["agg"] == "derived": - import ast as _ast - for node in _ast.walk(_ast.parse(m["expr"], mode="eval")): - if isinstance(node, _ast.Name): - need(node.id) - else: - if k not in base: - base.append(k) - for k in keys: - need(k) - return base, list(keys) - - -def _post_compute(row, key): - m = _model()["metrics"][key] - if m["agg"] == "ratio": - num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"]) - return (num / den) if den else 0.0 - if m["agg"] == "derived": - return _eval_expr(m["expr"], lambda k: _post_compute(row, k)) - return row.get(key) or 0.0 - - -# A metric's declared `format` -> the grid field type the compiler and the cells both read. -# One mapping, so a new metric cannot arrive as an untyped column. -_FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"} - - -def _clean_tree(tree, cols): - """Run a filter tree through the SAME validator the client's view state goes through. - - `filter_sql`'s documented input contract is "feed me CLEANED trees" — it is faithful to the - TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's - job. The UI path always cleans (view state is sanitised on the way into the store), but - store_query/store_rows are callable directly, and an uncleaned tree is where the two - diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here. - - Normalising at the entry point makes that unreachable rather than merely unlikely, and - brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf - module with zero imports of its own, so this cannot cycle. - """ - from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD, - MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS) - - # REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per - # level, total nodes and depth by DROPPING the excess — correct on the client, where the - # cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil. - # On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then - # reports a row_count and scope-wide totals that look authoritative for a query the caller - # never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]] - # forbids. A caller that exceeds the caps gets an error, not a quietly different answer. - total = 0 - - def _walk(nodes, depth): - nonlocal total - nodes = list(nodes or []) - if len(nodes) > MAX_FILTER_SIBLINGS: - raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit " - f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the " - f"excess, which would silently widen the result.") - for node in nodes: - if not isinstance(node, dict): - continue - total += 1 - if isinstance(node.get('children'), list): - if depth >= MAX_FILTER_DEPTH: - raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; " - f"refusing rather than dropping the deepest group.") - _walk(node['children'], depth + 1) - - _walk(tree, 1) - if total > MAX_FILTER_NODES: - raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. " - f"Refusing rather than truncating, which would silently widen it.") - - # CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by - # harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning - # on this path, and both ways it could arrive here are silent: - # - on a topic without that column, `clean_filter_tree` DROPS it as unknown — widening the - # query while store_rows reports an authoritative row_count for something nobody asked - # for (the same class as the sibling/depth caps above); - # - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a - # per-LINE predicate with the window silently discarded — a different question answered - # confidently, which is worse than dropping it. - # The discriminator is the `window`, not the column name: no column condition has one. - windowed = [] - cohorts = [] - - def _find(nodes): - for node in nodes or []: - if not isinstance(node, dict): - continue - if isinstance(node.get('children'), list): - _find(node['children']) - elif node.get('window') is not None: - windowed.append(str(node.get('colId'))) - elif node.get('colId') == _COHORT_FIELD: - cohorts.append(str(node.get('value'))) - - _find(tree) - # Owner item 5, and the same refusal for the same reason. A cohort leaf names a - # hand-curated set of CUSTOMERS held per user in the tenant store — this path knows nothing - # about it, and `clean_filter_tree` below would drop it as an unknown key, widening the - # query while store_rows still reported an authoritative row_count. If a caller ever needs - # it here, the fix is to pass the membership in, not to let it fall through. - if cohorts: - raise ModelError( - f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} — " - f"cohort membership is host state, not a column of this topic. Refusing rather than " - f"dropping it, which would silently widen the result under an authoritative count.") - if windowed: - raise ModelError( - f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}" - f" — those are resolved to a set of ids by harness.measure_filter, not compiled into " - f"this query. Refusing rather than dropping or mis-compiling them, either of which " - f"would silently answer a different question under an authoritative count.") - return clean_filter_tree(tree, set(cols)) - - -def _measure_row_sql(m, alias): - """A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the - aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line — deriving it - from the metric rather than hand-writing the column is what keeps ONE definition of the - number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a - count/count_distinct metric is a property of a SET, not of a row.""" - if m.get("agg") != "sum": - return None - f = m.get("field") - flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None - core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}" - return f"-({core})" if m.get("negate") else core - - -def store_columns(topic, include_measures=True, grain="aggregate"): - """The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1). - - `grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw - field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE - rather than needing HAVING. count-style metrics are dropped — they have no row value. - - This is the seam between the grid's field contract and the semantic model: the grid speaks - field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say - what a key resolves to. Nothing about a topic leaks into filter_sql itself. - - -> the dim's NAME column (what the grid displays and what a user means when - they type "contains floral"), filtered PRE-aggregation in WHERE. Correct - even in a grouped query: every row of a group shares its group's name. - _id -> the raw id, for exact machine-keyed filtering. - date -> the topic's date column, so a grid can filter/sort it like any other field. - -> the aggregate expression, marked `aggregate` so the caller routes it to - HAVING rather than WHERE (see store_query — that path is not built yet). - """ - t = _model()["topics"].get(topic) - if not t or "store" not in t: - raise ModelError(f"topic {topic!r} has no store binding") - s = t["store"] - cols = {} - for key, d in (s.get("dims") or {}).items(): - cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False} - # Only expose a separate id column when the dim actually HAS one. `city` is its own - # name (col == name_col), so a `city_id` would be a text column typed int — filtering - # it numerically would quietly compare 0 against 0 for every row. - if d.get("name_col") and d["name_col"] != d["col"]: - cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False} - if s.get("date_col"): - cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False} - if include_measures: - for k, m in _model()["metrics"].items(): - if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"): - continue # ratio/derived are post-computed, not SQL - # The field TYPE comes from the metric's declared format, never a guess: `units` is - # format:int, and typing it currency would render 791,960 units as dollars. - ftype = _FORMAT_TYPE.get(m.get("format"), "currency") - if grain == "row": - row_sql = _measure_row_sql(m, s["alias"]) - if row_sql: - cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False} - else: - cols[k] = {"sql": _measure_sql(m, s["alias"]), - "type": ftype, "aggregate": True} - return cols - - -#: The ceiling for a GROUPED `store_query`. One row per group, so this bounds DIMENSION -#: CARDINALITY, not payload — a tenant would need 200,000 distinct customers (or products, or -#: cities) to reach it. Deliberately NOT unbounded: a runaway group-by should fail, not swap. -MAX_GROUPS = 200_000 - - -def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, - team_id=None, filters=None, sort=None, limit=1000, exclude_services=False, - filter_tree=None, filter_conj="and", today=None): - """The semantic query over the tenant store — the engine behind the Analyst's - run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model.""" - t = _model()["topics"].get(topic) - if not t or "store" not in t: - raise ModelError(f"topic {topic!r} has no store binding") - s = t["store"] - alias = s["alias"] - dims = s.get("dims") or {} - # ⭐ A GROUPED QUERY IS BOUNDED BY GROUPS, NOT BY ROWS — and the 5,000 row ceiling applied to - # both, which made it a SILENT TRUNCATION of the answer rather than of a payload. - # - # ⛔ MEASURED 2026-08-09 on the Royal mirror, grouping 256,810 order lines by customer: - # limit=100 -> 100 groups, total 1,644,181.65 - # limit=1000 -> 1000 groups, total 9,042,614.10 - # limit=5000 -> 1748 groups, total 14,567,929.72 - # The TOTAL MOVES WITH THE CAP. A row window is honest because counts and totals are computed - # over the full scope beside it (`store_rows`' whole design); a GROUP window has no such - # companion — the groups ARE the answer, so dropping one is dropping data with nothing to - # notice. Royal has 1,943 customers so it passes today and would have passed every test, - # then gone quietly wrong for the first tenant with more ([[no-unverifiable-aggregates]]). - # - # ⚠ The ceiling is not removed, because unbounded is its own failure. It is raised to a bound - # no realistic dimension reaches, and — the load-bearing half — truncation is now a FACT the - # caller can read rather than something it must infer from `len(rows)`. - grouped = bool(group_by) - limit = max(1, min(int(limit or 1000), MAX_GROUPS if grouped else 5000)) - - base_keys, requested = _expand_measures(list(measures or [])) - if not base_keys: - raise ModelError("at least one measure required") - # Cross-topic base metrics (e.g. aov = revenue ÷ orders, where orders lives on sales_orders): - # in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain — - # never count(*) on the wrong table); in a GROUPED query they are refused (split the request). - foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic] - if foreign and (group_by or grain): - raise ModelError(f"metrics {foreign} belong to another topic — cross-topic measures are " - f"scalar-only; run them against their own topic when grouping") - base_keys = [k for k in base_keys if k not in foreign] - foreign_vals = {} - for k in foreign: - sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from, - date_to=date_to, team_id=team_id) - foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0 - if not base_keys: # purely foreign scalar request - return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)", - "measures": requested, "group_by": [], "grain": None} - - select, group_cols, params = [], [], [] - gb = [g for g in (group_by or []) if g] - for g in gb: - if g not in dims: - raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})") - d = dims[g] - select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}") - group_cols.append(d["col"]) - if d.get("name_col"): - select.append(f"max({d['name_col']}) AS {g}") - if grain: - if grain not in ("month", "week", "day"): - raise ModelError("grain must be month|week|day") - select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period") - group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))") - for k in base_keys: - select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}") - - where = [render_scope(s["scope_sql"].strip())] - # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes — - # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date). - # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' < - # '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides. - if date_from: - where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") - params.append(f"{date_from} 00:00:00") - if date_to: - where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") - params.append(f"{date_to} 23:59:59") - if team_id: - if not s.get("team_col"): - raise ModelError(f"topic {topic!r} is company-level — it has no business-unit filter") - where.append(f"{s['team_col']} = ?"); params.append(int(team_id)) - if exclude_services and s.get("service_filter"): - where.append(render_scope(s["service_filter"])) - for dim_key, vals in (filters or {}).items(): - if dim_key not in dims: - raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})") - # id dims filter by int; TEXT dims (e.g. city) filter by the value itself — either way - # every VALUE stays parameterized (the whitelist covers identifiers only) - def _coerce(v): - try: - return int(v) - except (TypeError, ValueError): - return str(v) - ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])] - where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})") - params.extend(ids) - - # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape; - # this is the nested, 11-operator contract the table UI emits. Compiled by - # harness/filter_sql, which is held in lock-step with the TS engine and the validator by - # aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in - # positional order with `where`. - if filter_tree: - from harness import filter_sql as _fs - cols = store_columns(topic) - pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols, - today=today) - if pred is not None: - if pred.uses_aggregate: - bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate")) - raise ModelError( - f"filter_tree references measure(s) {bad} — a measure filter has to land in " - f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path " - f"because its consumer (CG-2, the line-grain sales table) does not aggregate. " - f"Filter on dims, or aggregate first and filter the result.") - where.append(pred.sql) - params.extend(pred.params) - - sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} " - f"WHERE {' AND '.join(where)}") - if group_cols: - sql += f" GROUP BY {', '.join(group_cols)}" - if sort: - key = sort.lstrip("-") - if key not in base_keys + gb + (["period"] if grain else []): - raise ModelError(f"sort {sort!r} must reference a selected measure/dim") - sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}" - elif grain: - sql += " ORDER BY period" - # ⚠ `limit + 1` — ONE extra row, so "did this truncate" is a FACT and not the guess - # `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap). - sql += f" LIMIT {limit + 1}" - - con = _store_con() - try: - cur = con.execute(sql, params) - cols = [d[0] for d in cur.description] - rows = [dict(zip(cols, r)) for r in cur.fetchall()] - finally: - con.close() - for row in rows: # ratio/derived post-compute per group - row.update(foreign_vals) # scalar cross-topic components - for k in requested: - if _model()["metrics"][k]["agg"] in ("ratio", "derived"): - row[k] = _post_compute(row, k) - if "period" in row and row["period"] is not None: - row["period"] = str(row["period"])[:10] - truncated = len(rows) > limit - if truncated: - rows = rows[:limit] - return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql, - "measures": requested, "group_by": gb, "grain": grain, - # ⛔ A CALLER THAT AGGREGATES THESE ROWS MUST CHECK THIS. For a grouped query the - # groups ARE the answer, so a truncated result is a WRONG NUMBER, not a short list. - "truncated": truncated} - - -def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None, - filter_conj="and", search=None, sorts=None, limit=200, offset=0, - aggregates=None, exclude_services=False, id_sql=None, member_ids=None, - today=None): - """ROW-grain fetch — the line-grain counterpart to store_query's aggregate path (CG-2). - - store_query answers "what is the number"; this answers "which rows". At line grain the - payload is the binding constraint (254,837 sale_order_lines against 1,550 customers — the - whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back - WINDOWED. - - A window is only honest if the counts and totals are not windowed with it - ([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its - own query over the FULL scope, never from `len(rows)`: - - row_count rows matching the filter across the whole scope -> the N of "N of M" - total_count rows in the scope with no filter at all -> the M - aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope - - That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told - how many rows exist and shown totals for all of them, while receiving only the page they - can see. - - Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what - the client engine means at customer grain — that equivalence is what verify_filter_engine.py - exists to hold. - - Dates are returned already truncated to the 10-char ISO shape the filter compares against, - so what is displayed and what is filtered are the same string. Numerics come back RAW; the - payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding. - """ - t = _model()["topics"].get(topic) - if not t or "store" not in t: - raise ModelError(f"topic {topic!r} has no store binding") - from harness import filter_sql as _fs - s = t["store"] - alias = s["alias"] - cols = store_columns(topic, grain="row") - id_sql = id_sql or f"{alias}.id" - limit = max(1, min(int(limit or 200), 5000)) - offset = max(0, int(offset or 0)) - - # --- the scope every one of the three queries shares ------------------------------- - scope, scope_params = [render_scope(s["scope_sql"].strip())], [] - if date_from: - scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") - scope_params.append(f"{date_from} 00:00:00") - if date_to: - scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") - scope_params.append(f"{date_to} 23:59:59") - if team_id: - if not s.get("team_col"): - raise ModelError(f"topic {topic!r} is company-level — it has no business-unit filter") - scope.append(f"{s['team_col']} = ?") - scope_params.append(int(team_id)) - if exclude_services and s.get("service_filter"): - scope.append(render_scope(s["service_filter"])) - - # --- the user's narrowing, on top of the scope -------------------------------------- - narrow, narrow_params = [], [] - pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols, - today=today, - member_ids=member_ids or None, id_sql=id_sql) - if pred is not None: - if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway - raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}") - narrow.append(pred.sql) - narrow_params.extend(pred.params) - spred = _fs.compile_search(search, cols) - if spred is not None: - narrow.append(spred.sql) - narrow_params.extend(spred.params) - - frm = f"{s['table']} {alias} {s.get('join', '')}" - scope_where = " AND ".join(scope) - all_where = " AND ".join(scope + narrow) - con = _store_con() - try: - # 1. the WINDOW of rows - select = [f"{id_sql} AS _rid"] + [ - (f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date" - else f"{c['sql']} AS {k}") - for k, c in cols.items()] - order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC" - sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} " - f"ORDER BY {order} LIMIT {limit} OFFSET {offset}") - cur = con.execute(sql, scope_params + narrow_params) - names = [d[0] for d in cur.description] - rows = [dict(zip(names, r)) for r in cur.fetchall()] - - # 2. the two counts — over the WHOLE scope, never len(rows) - count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}" - row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0] - total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}", - scope_params).fetchone()[0] - - # 3. the aggregates — the topic's OWN metric definitions over the whole FILTERED scope - mts = _model()["metrics"] - keys = [k for k in (aggregates if aggregates is not None - else [k for k, m in mts.items() - if m.get("topic") == topic and m.get("agg") == "sum"])] - aggs = {} - if keys: - for k in keys: - if k not in mts or mts[k].get("topic") != topic: - raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}") - agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys) - + f" FROM {frm} WHERE {all_where}") - cur = con.execute(agg_sql, scope_params + narrow_params) - aggs = dict(zip([d[0] for d in cur.description], cur.fetchone())) - finally: - con.close() - - return { - "topic": topic, - "rows": rows, - "columns": {k: {"type": c["type"]} for k, c in cols.items()}, - "window": {"offset": offset, "limit": limit, "returned": len(rows)}, - "row_count": row_count, # N — matches the filter, across the WHOLE scope - "total_count": total_count, # M — the unfiltered scope - "aggregates": aggs, # over the whole FILTERED scope, not the window - "sql": sql, - "count_sql": count_sql, - } - - -def store_field_values(topic, dim, search=None, limit=25): - """Resolve real filterable values for a dim (the Analyst's get_field_values tool — fixes - 'NYC Florist' vs the actual record name BEFORE querying).""" - t = _model()["topics"].get(topic) - if not t or "store" not in t: - raise ModelError(f"topic {topic!r} has no store binding") - d = (t["store"].get("dims") or {}).get(dim) - if not d: - raise ModelError(f"unknown dim {dim!r} for topic {topic!r}") - if not d.get("name_col"): - raise ModelError(f"dim {dim!r} has no name column (filter by id)") - s = t["store"] - sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name " - f"FROM {s['table']} {s['alias']} {s.get('join','')} " - f"WHERE {render_scope(s['scope_sql'].strip())}") - params = [] - if search: - sql += f" AND {d['name_col']} ILIKE ?" - params.append(f"%{search}%") - sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}" - con = _store_con() - try: - return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()] - finally: - con.close() - - -def store_parity(date_from=None, date_to=None): - """THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope.""" - import core.periods as P - if not date_from: - date_from, date_to = P.ytd() - out = [] - for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced", - "revenue_invoiced", "orders_invoiced"): # wave 21 R2 — both filtered paths - m = _model()["metrics"][key] - # ⚠ A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a - # team is asked of a topic with no team_col, and the live builder refuses too — so - # looping the BUs here would fail the gate for a topic that is correct. Parity for a - # company-level number means company-level parity; pretending otherwise would either - # crash or, worse, compare two numbers that silently ignored the team. - _store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {} - scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,) - for tid in scopes: - live = metric(key, date_from, date_to, tid) - res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid) - ours = res["rows"][0][key] if res["rows"] else 0 - gap = abs((ours or 0) - (live or 0)) - out.append({"check": f"store.{key} == live.{key} (team={tid})", - "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live}) - # The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the - # agent's FIRST-agent book — a fully independent path (live partner read + line-domain sum) - # against the store's res_partner.agent_id attribution. Largest book = the sharpest check. - recs = O.search_read("res.partner", [("active", "in", [True, False]), - ("agent_ids", "!=", False)], ["agent_ids"], limit=100000) - books = {} - for r in recs: - a = (r.get("agent_ids") or [None])[0] - if a: - books.setdefault(a, []).append(r["id"]) - if books: - top = max(books, key=lambda k: len(books[k])) - live = O.sum_field("sale.order.line", - O.sale_line_domain(date_from, date_to, partner_ids=books[top]), - "price_subtotal") - res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to, - filters={"agent": [top]}) - ours = res["rows"][0]["revenue"] if res["rows"] else 0 - gap = abs((ours or 0) - (live or 0)) - out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, " - f"{len(books[top])} customers)", - "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live}) - return out +"""harness/semantic.py — the semantic seam (OM-0a, 2026-07-11). + +The ONE governed registry of topics + metrics (platform/model/*.yml) and the resolver that +turns a metric key into a number through the SAME proven data layer the validated modules use +(core/odoo domain builders + aggregate helpers) — so scope parity is by construction, and every +metric carries its definition, ai_context, and an independent validate contract. + +Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved- +dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4: +list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan: +.claude/wiki/research/omni-adoption.md (Part IV + the productization addendum). + +Design constraints honored (from /odoo-api): + - read_group dot-path GROUPBY fails → scalar aggregates here use sum_field/distinct_count; + grouped queries (OM-3+) must group by direct m2o fields only. + - Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...); + the YAML `scope:` block is the declarative statement of what that builder bakes in. + - READ-ONLY on Odoo, always. +""" +import ast +import functools +import operator +from pathlib import Path + +import yaml + +import core.odoo as O + +MODEL_DIR = Path(__file__).resolve().parents[1] / "model" + +# Topic scope binds to a NAMED, reviewed domain builder — never a YAML-assembled domain (one +# source of truth for scope until OM-2 makes topics executable against the store). +def _order_domain(*a, **kw): + # lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store + import modules.sales as S + return S.order_domain(*a, **kw) + + +def _customer_move_domain(move_type): + """A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly. + + ⚠ COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry + team_id = 1, so a BU filter on the document is meaningless — silently dropping the argument + would hand a business-unit user the company's number under their own name, which is the + widening sin in a different currency. + """ + def build(date_from=None, date_to=None, team_id=None, **kw): + if team_id: + raise ModelError( + f"customer {move_type} documents are COMPANY-LEVEL — credit notes and invoices " + f"carry no trustworthy business-unit tag, so a team-scoped total cannot be " + f"produced. Refusing rather than returning the company number.") + dom = [("move_type", "=", move_type), ("state", "=", "posted")] + if date_from: + dom.append(("invoice_date", ">=", date_from)) + if date_to: + dom.append(("invoice_date", "<=", date_to)) + return dom + return build + + +_BUILDERS = { + "sale_line_domain": O.sale_line_domain, + "sale_order_domain": _order_domain, + "credit_note_domain": _customer_move_domain("out_refund"), + "customer_invoice_domain": _customer_move_domain("out_invoice"), +} + + +class ModelError(Exception): + """A model-file problem (unknown key, bad reference, unparseable expr).""" + + +# ---------------------------------------------------------------- loading + +@functools.lru_cache(maxsize=1) +def _tenant(): + f = MODEL_DIR / "tenant.yml" + return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}} + + +def _sql_value(v): + """Render a tenant param into SQL safely (params are versioned config, not user input — + but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped.""" + if isinstance(v, bool): + raise ModelError("boolean tenant params not supported in scope_sql") + if isinstance(v, (int, float)): + return str(v) + if isinstance(v, (list, tuple)): + return ", ".join(_sql_value(x) for x in v) + return "'" + str(v).replace("'", "''") + "'" + + +def render_scope(sql): + """Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml.""" + params = _tenant().get("params") or {} + out = sql + for k, v in params.items(): + out = out.replace("{" + k + "}", _sql_value(v)) + if "{" in out and "}" in out: + import re as _re + missing = _re.findall(r"\{([a-z_]+)\}", out) + if missing: + raise ModelError(f"scope_sql references unknown tenant params: {missing}") + return out + + +@functools.lru_cache(maxsize=1) +def _model(): + topics, metrics = {}, {} + for f in sorted((MODEL_DIR / "topics").glob("*.yml")): + d = yaml.safe_load(f.read_text(encoding="utf-8")) + if not d or "key" not in d: + raise ModelError(f"topic file {f.name} lacks a 'key'") + if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS: + raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}") + # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) — metric() raises + # a clear error if a live resolution is attempted on it + topics[d["key"]] = d + for f in sorted((MODEL_DIR / "metrics").glob("*.yml")): + d = yaml.safe_load(f.read_text(encoding="utf-8")) + topic = d.get("topic") + if topic not in topics: + raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}") + for m in d.get("metrics", []): + if "key" not in m: + raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'") + if m["key"] in metrics: + raise ModelError(f"duplicate metric key {m['key']!r}") + m.setdefault("topic", topic) # a metric may name its own topic (rare) + if m["topic"] not in topics: + raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}") + metrics[m["key"]] = m + return {"topics": topics, "metrics": metrics} + + +def reload_model(): + _model.cache_clear() + _tenant.cache_clear() + # ⚠ The entity-measure offer is derived from the same files and would otherwise survive a + # reload — a stale offer is a column list that disagrees with the model it claims to come from. + _ENTITY_OFFER_CACHE.clear() + return _model() + + +def topics(): + return dict(_model()["topics"]) + + +def metrics(): + return dict(_model()["metrics"]) + + +def describe(key): + """Full metadata for a metric (the dictionary page / AI describe_topic feed).""" + m = _model()["metrics"].get(key) + if not m: + raise ModelError(f"unknown metric {key!r}") + return {**m, "topic_def": _model()["topics"][m["topic"]]} + + +# ---------------------------------------------------------------- resolving + +def _domain(topic_def, date_from, date_to, team_id): + b = topic_def.get("domain_builder") + if not b: + raise ModelError(f"topic {topic_def['key']!r} is store-only — use store_query() " + f"(no live-path domain builder bound yet)") + return _BUILDERS[b](date_from, date_to, team_id=team_id) + + +_OPS = {ast.Add: operator.add, ast.Sub: operator.sub, + ast.Mult: operator.mul, ast.Div: operator.truediv} + + +def _eval_expr(expr, resolve): + """Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens.""" + def ev(node): + if isinstance(node, ast.Expression): + return ev(node.body) + if isinstance(node, ast.BinOp) and type(node.op) in _OPS: + return _OPS[type(node.op)](ev(node.left), ev(node.right)) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return -ev(node.operand) + if isinstance(node, ast.Num): # py<3.8 compat name; Constant below + return node.n + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): + return node.value + if isinstance(node, ast.Name): + return resolve(node.id) + raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}") + try: + return ev(ast.parse(expr, mode="eval")) + except SyntaxError as e: + raise ModelError(f"unparseable derived expr {expr!r}: {e}") + + +def metric(key, date_from=None, date_to=None, team_id=None, _seen=None): + """Resolve a registered metric to a number for a window (+optional BU). The only public + compute path — every surface that shows this metric goes through here.""" + _seen = _seen or set() + if key in _seen: + raise ModelError(f"circular metric reference at {key!r}") + _seen = _seen | {key} + m = _model()["metrics"].get(key) + if not m: + raise ModelError(f"unknown metric {key!r}") + agg = m.get("agg") + if agg == "ratio": + num = metric(m["numerator"], date_from, date_to, team_id, _seen) + den = metric(m["denominator"], date_from, date_to, team_id, _seen) + return (num / den) if den else 0.0 + if agg == "derived": + return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen)) + t = _model()["topics"][m["topic"]] + dom = _domain(t, date_from, date_to, team_id) + # Wave 21 R2 — the LIVE twin of `store_filter_sql`. A filtered metric must filter BOTH + # paths, or store_parity would compare a narrowed store number against an unfiltered live + # one and the "parity failure" would read as a data problem. YAML shape: + # live_domain: [[field, op, value], ...] — appended verbatim to the topic domain. + for c in (m.get("live_domain") or []): + dom = dom + [tuple(c)] + if agg == "sum": + # ⚠ `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a + # STORE-ONLY topic, so the live path never met it — the first metric that is both negated + # and parity-checked (`returns`) would otherwise compare a negative live number against a + # positive store one, and the "parity failure" would look like a data problem. + got = O.sum_field(t["entity"], dom, m["field"]) + return -got if m.get("negate") else got + if agg == "count_distinct": + return O.distinct_count(t["entity"], dom, m["field"]) + if agg == "count": + return O.get_odoo().search_count(t["entity"], dom) + raise ModelError(f"metric {key!r}: unknown agg {agg!r}") + + +# ---------------------------------------------------------------- validate contracts + +def _v_order_level_revenue(date_from, date_to, team_id): + """Independent check: line-level Σ price_subtotal == order-level Σ amount_untaxed.""" + import modules.sales as S + line = metric("revenue", date_from, date_to, team_id) + order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed") + return line, order + + +def _v_order_count(date_from, date_to, team_id): + """Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders. + (Confirmed orders with zero lines exist in the data — 2 found at first proof, 2026-07-11; + counting them explicitly makes the identity exact instead of hiding them in a tolerance, + and surfaces them as a data-health signal.)""" + import modules.sales as S + order_level = metric("orders", date_from, date_to, team_id) + dom = S.order_domain(date_from, date_to, team_id) + from_lines = O.distinct_count("sale.order.line", + O.sale_line_domain(date_from, date_to, team_id), "order_id") + empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)]) + return order_level, from_lines + empty + + +def _v_gl_opex(date_from, date_to, team_id): + """Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted, + expense-type accounts), to the cent. team_id N/A (company-level).""" + res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to) + ours = res["rows"][0]["opex"] if res["rows"] else 0 + dom = [("move_id.state", "=", "posted"), + ("account_id.account_type", "in", ["expense", "expense_depreciation"])] + if date_from: + dom.append(("date", ">=", date_from)) + if date_to: + dom.append(("date", "<=", date_to)) + live = O.sum_field("account.move.line", dom, "balance") + return ours, live + + +def _v_ar_outstanding(date_from, date_to, team_id): + """Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals). + No date window — outstanding is an as-of-now balance.""" + res = store_query("receivables", ["ar_outstanding"]) + ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0 + live = O.sum_field("account.move", + [("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])], + "amount_residual_signed") + return ours, live + + +def _v_returns_sign(date_from, date_to, team_id): + """Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the + same posted credit notes, read straight from Odoo without going through the metric. + + A lost negation makes returns negative; a doubled one makes it negative again. Both read as + "no returns" rather than as an error, and neither is visible to any self-consistency check — + the store and the live path would agree with each other perfectly while both being wrong. + """ + ours = metric("returns", date_from, date_to, team_id) + raw = O.sum_field("account.move", + _BUILDERS["credit_note_domain"](date_from, date_to, team_id), + "amount_untaxed_signed") + return ours, abs(raw) + + +def _v_line_vs_move_grain(date_from, date_to, team_id): + """Independent check on the invoice-LINE topic: the store's netted product-line total must + equal the same figure read straight from LIVE Odoo. + + Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION — + `price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates + revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER — the + invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including + them changes the total silently rather than erroring. + + Company-level (team_id None) uses the strongest possible reference: the MOVE-grain + `amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue + (account.move carries no business unit — every invoice sits on team 1), so the reference is + the live line-level aggregate reached through the sale-order dot path: still an independent + engine and query path from the DuckDB store, which is what the contract requires. + """ + ours = (store_query("invoice_lines", ["invoiced_line_sales"], + date_from=date_from, date_to=date_to, team_id=team_id) + .get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0 + if team_id is None: + raw = O.sum_field("account.move", + [("state", "=", "posted"), + ("move_type", "in", ["out_invoice", "out_refund"]), + ("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)], + "amount_untaxed_signed") + return ours, raw + dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"), + ("date", ">=", date_from), ("date", "<=", date_to), + ("sale_line_ids.order_id.team_id", "=", team_id)] + gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")], + "price_subtotal") + refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")], + "price_subtotal") + return ours, (gross or 0) - (refunds or 0) + + +_VALIDATORS = { + "returns_sign": _v_returns_sign, + "line_vs_move_grain": _v_line_vs_move_grain, + "order_level_revenue": _v_order_level_revenue, + "order_count": _v_order_count, + "gl_opex": _v_gl_opex, + "ar_outstanding": _v_ar_outstanding, +} + + +def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01): + """Run a metric's declared independent cross-check. Returns the standard check dict + (the validate.py convention: check/ok/gap).""" + m = _model()["metrics"].get(key) + if not m: + raise ModelError(f"unknown metric {key!r}") + contract = m.get("validate") + if not contract: + return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0, + "note": "declare one in model/metrics when an independent aggregate exists"} + fn = _VALIDATORS.get(contract["method"]) + if not fn: + raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}") + ours, independent = fn(date_from, date_to, team_id) + gap = abs((ours or 0) - (independent or 0)) + return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})", + "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent} + + +def validate(pre=None): + """Module-convention validate(): run every declared contract over YTD, all-BU + per-BU.""" + import core.periods as P + yf, yt = P.ytd() + out = [] + for key, m in _model()["metrics"].items(): + if not m.get("validate"): + continue + out.append(validate_metric(key, yf, yt, None)) + topic = _model()["topics"][m["topic"]] + if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs + for tid in O.TEAM_IDS: + out.append(validate_metric(key, yf, yt, tid)) + return out + + +# ---------------------------------------------------------------- store path (OM-2) +# The compiler behind run_semantic_query: registered metrics + whitelisted dims → DuckDB SQL over +# the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files +# (trusted); every VALUE is parameterized — the small model only ever picks keys, so there is no +# injection surface. Live XML-RPC stays the validate() path (the store never validates itself). + +def _store_con(): + import harness.datastore as DS + if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently + raise ModelError("the data cache is still warming up after a restart (a one-time first " + "sync) — the dashboards work now from live data; ask me again in a " + "moment and I'll have it") + return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention + + +def _measure_sql(m, alias): + """SQL for a base measure. Supports FILTERED measures (`store_filter_sql` — the Omni + filtered-measure concept: sum(CASE WHEN … )) and `negate: true` (e.g. GL income, stored + credit-negative, reported positive).""" + agg, f = m.get("agg"), m.get("field") + flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None + if agg == "sum": + core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})" + elif agg == "count_distinct": + core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt + else f"count(DISTINCT {alias}.{f})") + elif agg == "count": + core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)" + else: + raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression") + return f"-({core})" if m.get("negate") else core + + +def _expand_measures(keys): + """Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the + requested keys for post-compute. Returns (base_keys, requested_keys).""" + mts = _model()["metrics"] + base, seen = [], set() + + def need(k): + if k in seen: + return + seen.add(k) + m = mts.get(k) + if not m: + raise ModelError(f"unknown metric {k!r}") + if m["agg"] == "ratio": + need(m["numerator"]); need(m["denominator"]) + elif m["agg"] == "derived": + import ast as _ast + for node in _ast.walk(_ast.parse(m["expr"], mode="eval")): + if isinstance(node, _ast.Name): + need(node.id) + else: + if k not in base: + base.append(k) + for k in keys: + need(k) + return base, list(keys) + + +def _post_compute(row, key): + m = _model()["metrics"][key] + if m["agg"] == "ratio": + num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"]) + return (num / den) if den else 0.0 + if m["agg"] == "derived": + return _eval_expr(m["expr"], lambda k: _post_compute(row, k)) + return row.get(key) or 0.0 + + +# A metric's declared `format` -> the grid field type the compiler and the cells both read. +# One mapping, so a new metric cannot arrive as an untyped column. +_FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"} + + +def _clean_tree(tree, cols): + """Run a filter tree through the SAME validator the client's view state goes through. + + `filter_sql`'s documented input contract is "feed me CLEANED trees" — it is faithful to the + TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's + job. The UI path always cleans (view state is sanitised on the way into the store), but + store_query/store_rows are callable directly, and an uncleaned tree is where the two + diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here. + + Normalising at the entry point makes that unreachable rather than merely unlikely, and + brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf + module with zero imports of its own, so this cannot cycle. + """ + from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD, + MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS) + + # REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per + # level, total nodes and depth by DROPPING the excess — correct on the client, where the + # cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil. + # On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then + # reports a row_count and scope-wide totals that look authoritative for a query the caller + # never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]] + # forbids. A caller that exceeds the caps gets an error, not a quietly different answer. + total = 0 + + def _walk(nodes, depth): + nonlocal total + nodes = list(nodes or []) + if len(nodes) > MAX_FILTER_SIBLINGS: + raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit " + f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the " + f"excess, which would silently widen the result.") + for node in nodes: + if not isinstance(node, dict): + continue + total += 1 + if isinstance(node.get('children'), list): + if depth >= MAX_FILTER_DEPTH: + raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; " + f"refusing rather than dropping the deepest group.") + _walk(node['children'], depth + 1) + + _walk(tree, 1) + if total > MAX_FILTER_NODES: + raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. " + f"Refusing rather than truncating, which would silently widen it.") + + # CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by + # harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning + # on this path, and both ways it could arrive here are silent: + # - on a topic without that column, `clean_filter_tree` DROPS it as unknown — widening the + # query while store_rows reports an authoritative row_count for something nobody asked + # for (the same class as the sibling/depth caps above); + # - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a + # per-LINE predicate with the window silently discarded — a different question answered + # confidently, which is worse than dropping it. + # The discriminator is the `window`, not the column name: no column condition has one. + windowed = [] + cohorts = [] + + def _find(nodes): + for node in nodes or []: + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + _find(node['children']) + elif node.get('window') is not None: + windowed.append(str(node.get('colId'))) + elif node.get('colId') == _COHORT_FIELD: + cohorts.append(str(node.get('value'))) + + _find(tree) + # Owner item 5, and the same refusal for the same reason. A cohort leaf names a + # hand-curated set of CUSTOMERS held per user in the tenant store — this path knows nothing + # about it, and `clean_filter_tree` below would drop it as an unknown key, widening the + # query while store_rows still reported an authoritative row_count. If a caller ever needs + # it here, the fix is to pass the membership in, not to let it fall through. + if cohorts: + raise ModelError( + f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} — " + f"cohort membership is host state, not a column of this topic. Refusing rather than " + f"dropping it, which would silently widen the result under an authoritative count.") + if windowed: + raise ModelError( + f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}" + f" — those are resolved to a set of ids by harness.measure_filter, not compiled into " + f"this query. Refusing rather than dropping or mis-compiling them, either of which " + f"would silently answer a different question under an authoritative count.") + return clean_filter_tree(tree, set(cols)) + + +def _measure_row_sql(m, alias): + """A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the + aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line — deriving it + from the metric rather than hand-writing the column is what keeps ONE definition of the + number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a + count/count_distinct metric is a property of a SET, not of a row.""" + if m.get("agg") != "sum": + return None + f = m.get("field") + flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None + core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}" + return f"-({core})" if m.get("negate") else core + + +def store_columns(topic, include_measures=True, grain="aggregate"): + """The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1). + + `grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw + field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE + rather than needing HAVING. count-style metrics are dropped — they have no row value. + + This is the seam between the grid's field contract and the semantic model: the grid speaks + field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say + what a key resolves to. Nothing about a topic leaks into filter_sql itself. + + -> the dim's NAME column (what the grid displays and what a user means when + they type "contains floral"), filtered PRE-aggregation in WHERE. Correct + even in a grouped query: every row of a group shares its group's name. + _id -> the raw id, for exact machine-keyed filtering. + date -> the topic's date column, so a grid can filter/sort it like any other field. + -> the aggregate expression, marked `aggregate` so the caller routes it to + HAVING rather than WHERE (see store_query — that path is not built yet). + """ + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + s = t["store"] + cols = {} + for key, d in (s.get("dims") or {}).items(): + cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False} + # Only expose a separate id column when the dim actually HAS one. `city` is its own + # name (col == name_col), so a `city_id` would be a text column typed int — filtering + # it numerically would quietly compare 0 against 0 for every row. + if d.get("name_col") and d["name_col"] != d["col"]: + cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False} + if s.get("date_col"): + cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False} + if include_measures: + for k, m in _model()["metrics"].items(): + if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"): + continue # ratio/derived are post-computed, not SQL + # The field TYPE comes from the metric's declared format, never a guess: `units` is + # format:int, and typing it currency would render 791,960 units as dollars. + ftype = _FORMAT_TYPE.get(m.get("format"), "currency") + if grain == "row": + row_sql = _measure_row_sql(m, s["alias"]) + if row_sql: + cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False} + else: + cols[k] = {"sql": _measure_sql(m, s["alias"]), + "type": ftype, "aggregate": True} + return cols + + +#: The ceiling for a GROUPED `store_query`. One row per group, so this bounds DIMENSION +#: CARDINALITY, not payload — a tenant would need 200,000 distinct customers (or products, or +#: cities) to reach it. Deliberately NOT unbounded: a runaway group-by should fail, not swap. +MAX_GROUPS = 200_000 + + +def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None, + team_id=None, filters=None, sort=None, limit=1000, exclude_services=False, + filter_tree=None, filter_conj="and", today=None): + """The semantic query over the tenant store — the engine behind the Analyst's + run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model.""" + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + s = t["store"] + alias = s["alias"] + dims = s.get("dims") or {} + # ⭐ A GROUPED QUERY IS BOUNDED BY GROUPS, NOT BY ROWS — and the 5,000 row ceiling applied to + # both, which made it a SILENT TRUNCATION of the answer rather than of a payload. + # + # ⛔ MEASURED 2026-08-09 on the Royal mirror, grouping 256,810 order lines by customer: + # limit=100 -> 100 groups, total 1,644,181.65 + # limit=1000 -> 1000 groups, total 9,042,614.10 + # limit=5000 -> 1748 groups, total 14,567,929.72 + # The TOTAL MOVES WITH THE CAP. A row window is honest because counts and totals are computed + # over the full scope beside it (`store_rows`' whole design); a GROUP window has no such + # companion — the groups ARE the answer, so dropping one is dropping data with nothing to + # notice. Royal has 1,943 customers so it passes today and would have passed every test, + # then gone quietly wrong for the first tenant with more ([[no-unverifiable-aggregates]]). + # + # ⚠ The ceiling is not removed, because unbounded is its own failure. It is raised to a bound + # no realistic dimension reaches, and — the load-bearing half — truncation is now a FACT the + # caller can read rather than something it must infer from `len(rows)`. + grouped = bool(group_by) + limit = max(1, min(int(limit or 1000), MAX_GROUPS if grouped else 5000)) + + base_keys, requested = _expand_measures(list(measures or [])) + if not base_keys: + raise ModelError("at least one measure required") + # Cross-topic base metrics (e.g. aov = revenue ÷ orders, where orders lives on sales_orders): + # in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain — + # never count(*) on the wrong table); in a GROUPED query they are refused (split the request). + foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic] + if foreign and (group_by or grain): + raise ModelError(f"metrics {foreign} belong to another topic — cross-topic measures are " + f"scalar-only; run them against their own topic when grouping") + base_keys = [k for k in base_keys if k not in foreign] + foreign_vals = {} + for k in foreign: + sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from, + date_to=date_to, team_id=team_id) + foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0 + if not base_keys: # purely foreign scalar request + return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)", + "measures": requested, "group_by": [], "grain": None} + + select, group_cols, params = [], [], [] + gb = [g for g in (group_by or []) if g] + for g in gb: + if g not in dims: + raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})") + d = dims[g] + select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}") + group_cols.append(d["col"]) + if d.get("name_col"): + select.append(f"max({d['name_col']}) AS {g}") + if grain: + if grain not in ("month", "week", "day"): + raise ModelError("grain must be month|week|day") + select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period") + group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))") + for k in base_keys: + select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}") + + where = [render_scope(s["scope_sql"].strip())] + # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes — + # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date). + # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' < + # '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides. + if date_from: + where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") + params.append(f"{date_from} 00:00:00") + if date_to: + where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") + params.append(f"{date_to} 23:59:59") + if team_id: + if not s.get("team_col"): + raise ModelError(f"topic {topic!r} is company-level — it has no business-unit filter") + where.append(f"{s['team_col']} = ?"); params.append(int(team_id)) + if exclude_services and s.get("service_filter"): + where.append(render_scope(s["service_filter"])) + for dim_key, vals in (filters or {}).items(): + if dim_key not in dims: + raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})") + # id dims filter by int; TEXT dims (e.g. city) filter by the value itself — either way + # every VALUE stays parameterized (the whitelist covers identifiers only) + def _coerce(v): + try: + return int(v) + except (TypeError, ValueError): + return str(v) + ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])] + where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})") + params.extend(ids) + + # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape; + # this is the nested, 11-operator contract the table UI emits. Compiled by + # harness/filter_sql, which is held in lock-step with the TS engine and the validator by + # aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in + # positional order with `where`. + if filter_tree: + from harness import filter_sql as _fs + cols = store_columns(topic) + pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols, + today=today) + if pred is not None: + if pred.uses_aggregate: + bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate")) + raise ModelError( + f"filter_tree references measure(s) {bad} — a measure filter has to land in " + f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path " + f"because its consumer (CG-2, the line-grain sales table) does not aggregate. " + f"Filter on dims, or aggregate first and filter the result.") + where.append(pred.sql) + params.extend(pred.params) + + sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} " + f"WHERE {' AND '.join(where)}") + if group_cols: + sql += f" GROUP BY {', '.join(group_cols)}" + if sort: + key = sort.lstrip("-") + if key not in base_keys + gb + (["period"] if grain else []): + raise ModelError(f"sort {sort!r} must reference a selected measure/dim") + sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}" + elif grain: + sql += " ORDER BY period" + # ⚠ `limit + 1` — ONE extra row, so "did this truncate" is a FACT and not the guess + # `len(rows) == limit` makes (which is wrong exactly when the count lands on the cap). + sql += f" LIMIT {limit + 1}" + + con = _store_con() + try: + cur = con.execute(sql, params) + cols = [d[0] for d in cur.description] + rows = [dict(zip(cols, r)) for r in cur.fetchall()] + finally: + con.close() + for row in rows: # ratio/derived post-compute per group + row.update(foreign_vals) # scalar cross-topic components + for k in requested: + if _model()["metrics"][k]["agg"] in ("ratio", "derived"): + row[k] = _post_compute(row, k) + if "period" in row and row["period"] is not None: + row["period"] = str(row["period"])[:10] + truncated = len(rows) > limit + if truncated: + rows = rows[:limit] + return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql, + "measures": requested, "group_by": gb, "grain": grain, + # ⛔ A CALLER THAT AGGREGATES THESE ROWS MUST CHECK THIS. For a grouped query the + # groups ARE the answer, so a truncated result is a WRONG NUMBER, not a short list. + "truncated": truncated} + + +def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None, + filter_conj="and", search=None, sorts=None, limit=200, offset=0, + aggregates=None, exclude_services=False, id_sql=None, member_ids=None, + today=None): + """ROW-grain fetch — the line-grain counterpart to store_query's aggregate path (CG-2). + + store_query answers "what is the number"; this answers "which rows". At line grain the + payload is the binding constraint (254,837 sale_order_lines against 1,550 customers — the + whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back + WINDOWED. + + A window is only honest if the counts and totals are not windowed with it + ([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its + own query over the FULL scope, never from `len(rows)`: + + row_count rows matching the filter across the whole scope -> the N of "N of M" + total_count rows in the scope with no filter at all -> the M + aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope + + That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told + how many rows exist and shown totals for all of them, while receiving only the page they + can see. + + Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what + the client engine means at customer grain — that equivalence is what verify_filter_engine.py + exists to hold. + + Dates are returned already truncated to the 10-char ISO shape the filter compares against, + so what is displayed and what is filtered are the same string. Numerics come back RAW; the + payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding. + """ + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + from harness import filter_sql as _fs + s = t["store"] + alias = s["alias"] + cols = store_columns(topic, grain="row") + id_sql = id_sql or f"{alias}.id" + limit = max(1, min(int(limit or 200), 5000)) + offset = max(0, int(offset or 0)) + + # --- the scope every one of the three queries shares ------------------------------- + scope, scope_params = [render_scope(s["scope_sql"].strip())], [] + if date_from: + scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?") + scope_params.append(f"{date_from} 00:00:00") + if date_to: + scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?") + scope_params.append(f"{date_to} 23:59:59") + if team_id: + if not s.get("team_col"): + raise ModelError(f"topic {topic!r} is company-level — it has no business-unit filter") + scope.append(f"{s['team_col']} = ?") + scope_params.append(int(team_id)) + if exclude_services and s.get("service_filter"): + scope.append(render_scope(s["service_filter"])) + + # --- the user's narrowing, on top of the scope -------------------------------------- + narrow, narrow_params = [], [] + pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols, + today=today, + member_ids=member_ids or None, id_sql=id_sql) + if pred is not None: + if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway + raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}") + narrow.append(pred.sql) + narrow_params.extend(pred.params) + spred = _fs.compile_search(search, cols) + if spred is not None: + narrow.append(spred.sql) + narrow_params.extend(spred.params) + + frm = f"{s['table']} {alias} {s.get('join', '')}" + scope_where = " AND ".join(scope) + all_where = " AND ".join(scope + narrow) + con = _store_con() + try: + # 1. the WINDOW of rows + select = [f"{id_sql} AS _rid"] + [ + (f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date" + else f"{c['sql']} AS {k}") + for k, c in cols.items()] + order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC" + sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} " + f"ORDER BY {order} LIMIT {limit} OFFSET {offset}") + cur = con.execute(sql, scope_params + narrow_params) + names = [d[0] for d in cur.description] + rows = [dict(zip(names, r)) for r in cur.fetchall()] + + # 2. the two counts — over the WHOLE scope, never len(rows) + count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}" + row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0] + total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}", + scope_params).fetchone()[0] + + # 3. the aggregates — the topic's OWN metric definitions over the whole FILTERED scope + mts = _model()["metrics"] + keys = [k for k in (aggregates if aggregates is not None + else [k for k, m in mts.items() + if m.get("topic") == topic and m.get("agg") == "sum"])] + aggs = {} + if keys: + for k in keys: + if k not in mts or mts[k].get("topic") != topic: + raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}") + agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys) + + f" FROM {frm} WHERE {all_where}") + cur = con.execute(agg_sql, scope_params + narrow_params) + aggs = dict(zip([d[0] for d in cur.description], cur.fetchone())) + finally: + con.close() + + return { + "topic": topic, + "rows": rows, + "columns": {k: {"type": c["type"]} for k, c in cols.items()}, + "window": {"offset": offset, "limit": limit, "returned": len(rows)}, + "row_count": row_count, # N — matches the filter, across the WHOLE scope + "total_count": total_count, # M — the unfiltered scope + "aggregates": aggs, # over the whole FILTERED scope, not the window + "sql": sql, + "count_sql": count_sql, + } + + +def store_field_values(topic, dim, search=None, limit=25): + """Resolve real filterable values for a dim (the Analyst's get_field_values tool — fixes + 'NYC Florist' vs the actual record name BEFORE querying).""" + t = _model()["topics"].get(topic) + if not t or "store" not in t: + raise ModelError(f"topic {topic!r} has no store binding") + d = (t["store"].get("dims") or {}).get(dim) + if not d: + raise ModelError(f"unknown dim {dim!r} for topic {topic!r}") + if not d.get("name_col"): + raise ModelError(f"dim {dim!r} has no name column (filter by id)") + s = t["store"] + sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name " + f"FROM {s['table']} {s['alias']} {s.get('join','')} " + f"WHERE {render_scope(s['scope_sql'].strip())}") + params = [] + if search: + sql += f" AND {d['name_col']} ILIKE ?" + params.append(f"%{search}%") + sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}" + con = _store_con() + try: + return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()] + finally: + con.close() + + +# --- ENTITY LOOKBACK MEASURES (W37-T10 / owner item 4, ruling R1) ------------------------- +# +# ⭐⭐ THE FEATURE, STATED ONCE: a database that is a REGISTRY (`odoo_products`, `odoo_agents` — +# a catalogue, no date column of its own) gets columns that measure a FACT topic over a window +# and key the answer back to its own rows. "Revenue, last 90 days" as a column on the SKU grid. +# +# ⛔ WHY THIS IS NOT `core.measure_resolve`. That family is CUSTOMER-GRAIN by construction — it +# resolves against `allowed_pids` that ARE partner ids, through `harness.measure_filter`, whose +# `_entity_sql` names the customer entity. Neither file is in this wave's fences, so generalising +# them was not available; this is the entity-neutral path beside it, and the duplication is +# DELIBERATE and BOOKED (see the wave-37 lane-B mailbox, ownership-gap learning). The day one +# lane owns both, `measure_resolve.column_values` should become a thin caller of this. +# +# ⛔ AN OFFER IS A PROMISE. `_rollup_source_offer`'s docstring makes this argument for rollups and +# it is the same argument here: a measure this returns but cannot answer mints a column that sits +# BLANK forever looking configured — no error, nothing to notice. So `entity_measures()` refuses +# a key it cannot prove, rather than passing the model's list through. + + +class _EntityMeasureError(ModelError): + """A binding problem, not a data problem — raised at OFFER time so it never reaches a cell.""" + + +def topic_for_grid(grid_key): + """The ENTITY topic that describes the database `grid_key`, or None. + + ⭐ THE BINDING IS ALREADY DECLARED AT BOTH ENDS and this reads it rather than adding a third + place to state it: `odoo_products.yml` carries `grid: product_data`, `odoo_agents.yml` carries + `grid: ut_odoo_agents` (W33-T46's own convention — *"the same store key the nav opens"*). A + hand-written `{table_key: topic}` map in a route would be a second definition of one fact and + would drift the day a topic is renamed. + """ + key = str(grid_key or "").strip() + if not key: + return None + for tkey, t in _model()["topics"].items(): + if str(t.get("grid") or "") == key: + return tkey + return None + + +def entity_measure_bindings(topic): + """`[{source, dim, keys, not_yet}, …]` — every FACT topic this entity draws columns from. + + ⭐⭐ A LIST, NOT ONE BINDING, since W37-T13. An entity's columns legitimately come from more + than one fact topic: the product grid takes revenue/units/margin from `sales_lines` and stock + in/out from `stock_moves`, and those are different tables joined by different dims. Collapsing + them into one source would have forced stock movement into the sales topic, where it has no + row — which is how a metric ends up defined twice. + + ⚠ BOTH SHAPES PARSE. A topic may declare `measures:` as a single mapping (the wave's original + form, still used by `odoo_agents`) or as a LIST of them. Accepting only the list would have + been a silent break of every binding written before this change. + + Structural read only: this says what the model CLAIMS. `entity_measures()` is what proves it. + """ + t = _model()["topics"].get(topic) + if not t: + raise ModelError(f"unknown topic {topic!r}") + raw = t.get("measures") + if isinstance(raw, dict): + raw = [raw] + if not isinstance(raw, list): + return [] + out = [] + for b in raw: + if not isinstance(b, dict) or not b.get("source") or not b.get("dim"): + continue + out.append({"source": str(b["source"]), "dim": str(b["dim"]), + "keys": list(b.get("keys") or []), "not_yet": list(b.get("not_yet") or [])}) + return out + + +def entity_measure_binding(topic): + """The FIRST binding, or None — kept for callers that predate the multi-source shape.""" + bs = entity_measure_bindings(topic) + return bs[0] if bs else None + + +def entity_measures(topic, strict=False): + """The measure CATALOGUE an entity topic may offer -> `[{key,label,type,format,empty,…}]`. + + The shape is the workspace `measures` contract (`{key, label, type}` — `customer-grid/types.ts` + `Measure`), with the extra keys a resolver needs carried alongside; the client reads the three + it knows and ignores the rest. + + ⛔ FOUR REFUSALS, each one a column that would otherwise render blank forever: + 1. the metric does not exist, or does not live on the SOURCE topic — a foreign base measure + is refused by `store_query` the moment a `group_by` is present, so offering it mints a + column that can only ever error (this is exactly what excludes `aov` and `orders`); + 2. the dim is not a dim of the source topic — nothing to group by; + 3. the metric declares no `empty:` family — C1: a metric that cannot say what a row with no + activity renders as does not ship, because the default IS a claim about truth; + 4. the metric's format has no grid type — `MEASURE_FIELD_TYPES` is {currency,int,pct}, so a + date- or text-valued measure cannot render as a column whatever we promise. + + `strict=True` raises on the first refusal (the gate's mode). Otherwise a refused key is simply + absent from the offer and its reason is available through `entity_measure_refusals`. + """ + return _entity_offer(topic, strict=strict)[0] + + +def entity_measure_refusals(topic): + """`[{key, reason}]` — every declared key `entity_measures` would NOT offer, and why. + + ⭐ The reporting half of standing rule 1 applied to a catalogue: a key that drops out has a + stated cause rather than simply being missing from a list nobody diffs. + """ + return _entity_offer(topic)[1] + + +#: The offer is a pure function of the MODEL FILES, and `_model()` is already `lru_cache`d — so +#: this caches the derivation, not the read. It matters because `routes_grid::events` rebuilds the +#: product assembly on EVERY write event (hide a field, save a view, patch a cell), and W30-T30 +#: records what putting real work on that path costs. Cleared by `reload_model()` like every other +#: model-derived cache, so a yml edit is not stale until a restart. +_ENTITY_OFFER_CACHE = {} + + +def _entity_offer(topic, strict=False): + if not strict and topic in _ENTITY_OFFER_CACHE: + return _ENTITY_OFFER_CACHE[topic] + out = _entity_offer_uncached(topic, strict=strict) + # ⛔⛔ AN INDETERMINATE ANSWER IS NEVER CACHED, and this is the more dangerous half of the + # guard above. The cache lives for the PROCESS, so a cold start that could not reach the store + # would freeze its "I don't know" into the offer every later request reads — long after the + # mirror came up. Refusing is safe; refusing FOREVER because of one early request is not. + # ⚠ The symmetric error is just as bad in the other direction: cache an over-broad offer taken + # during warmup and the column is promised for the process lifetime. + if not strict and not any(r.get("indeterminate") for r in out[1]): + _ENTITY_OFFER_CACHE[topic] = out + return out + + +def _entity_offer_uncached(topic, strict=False): + out, refused = [], [] + seen = set() + for b in entity_measure_bindings(topic): + o, r = _one_binding_offer(topic, b, strict=strict) + # ⛔ FIRST BINDING WINS A DUPLICATE KEY, and the collision is REPORTED rather than + # silently resolved: two fact topics offering the same metric key would otherwise give a + # column whose source depends on dict order. + for m in o: + if m["key"] in seen: + refused.append({"key": m["key"], + "reason": f"already offered by another source topic; " + f"{b['source']}'s copy is not used"}) + continue + seen.add(m["key"]) + out.append(m) + refused.extend(r) + return out, refused + + +def _source_ready(name): + """Is this mirror table present AND finished backfilling? `None` when it cannot be asked. + + ⛔ AN OFFER IS A PROMISE, AND A MODEL BINDING IS NOT EVIDENCE THE DATA IS THERE. A metric can + resolve perfectly in the model and still have no rows to group, because the mirror is synced + per ENTITY and a Space hydrates from a snapshot taken before the entity existed — which is + exactly the window W37-T13 opens by adding `stock_move`. Offering a column then would mint a + Metric field that renders BLANK forever with no error anywhere, this repo's most repeated + defect shape. + """ + # ⛔ TWO CONDITIONS, NOT ONE. A table can EXIST and be half-backfilled — `_ensure_table` runs + # before the first batch — and a grouped query over a partial table returns totals that are + # wrong in the one direction nobody checks: too small, per row, with no error. `entity_live` + # is the per-entity phase, which `ready()` deliberately no longer covers for an optional + # entity (see its comment: covering it would make adding one an outage). + try: + import harness.datastore as _ds + # ⛔⛔ "THERE IS NO STORE" IS NOT "THIS ENTITY IS NOT SYNCED", AND CONFLATING THEM CACHES + # AN EMPTY OFFER FOR THE PROCESS. `entity_live` answers False for both, so without this + # line a cold container with no mirror yet refuses EVERY binding, that refusal is + # definite rather than indeterminate, and it is memoised — so the grid serves no measures + # at all long after the mirror arrives. Measured: with the store path pointed at a file + # that does not exist, the offer came back `[]` AND was cached. + if not _ds.DB_PATH.exists() or not _ds.ready(): + return None # cannot ask -> INDETERMINATE, and never cached + if not _ds.entity_live(name): + return False # asked, and this entity really is not live yet + con = _store_con() + except Exception: # noqa: BLE001 + return None # cannot ask -> INDETERMINATE + try: + con.execute(f"SELECT 1 FROM {name} LIMIT 0") + return True + except Exception: # noqa: BLE001 + return False + finally: + try: + con.close() + except Exception: # noqa: BLE001 + pass + + +def _one_binding_offer(topic, b, strict=False): + src = _model()["topics"].get(b["source"]) + if not src or "store" not in src: + raise _EntityMeasureError( + f"{topic}: measure source {b['source']!r} has no store binding") + dims = (src["store"].get("dims") or {}) + if b["dim"] not in dims: + raise _EntityMeasureError( + f"{topic}: measure dim {b['dim']!r} is not a dim of {b['source']!r} " + f"(allowed: {sorted(dims)})") + # ⛔ THE TABLE CHECK, before any key is offered from this binding. + _tbl = (src["store"] or {}).get("table") + _rdy = _source_ready(_tbl) if _tbl else True + if _rdy is not True: + # ⛔ `None` REFUSES TOO, and that is the opposite of the obvious reading. `None` means "the + # store could not be asked" — a warming or absent mirror — and a store that cannot answer + # THIS question cannot answer the grouped query either, so offering the column would mint + # exactly the permanently-blank field the check exists to prevent. Fail closed. + why = ((f"the mirror has not finished syncing `{_tbl}`, so every key from {b['source']!r} " + f"would render blank or — worse — as a too-small number from a half-filled table; " + f"run `datastore.sync_entity('{_tbl}')` until its phase is `live`") + if _rdy is False else + (f"the tenant store could not be read, so whether `{_tbl}` can answer is UNKNOWN; " + f"refusing rather than promising a column that may render blank")) + if strict: + raise _EntityMeasureError(f"{topic}: {why}") + return [], [{"key": k, "reason": why, "indeterminate": _rdy is None} + for k in b["keys"]] + + mts = _model()["metrics"] + out, refused = [], [] + + def refuse(key, reason): + if strict: + raise _EntityMeasureError(f"{topic}: measure {key!r} refused — {reason}") + refused.append({"key": key, "reason": reason}) + + for key in b["keys"]: + m = mts.get(key) + if not m: + refuse(key, "no such metric in the model") + continue + # ⛔ THE CROSS-TOPIC TEST IS ON THE EXPANDED BASE, NOT THE METRIC'S OWN `topic`. `aov` + # declares no topic of its own and would pass a naive check; its DENOMINATOR `orders` + # lives on `sales_orders`, and that is what `store_query` refuses under a group_by. + try: + base, _ = _expand_measures([key]) + except ModelError as e: + refuse(key, str(e)) + continue + foreign = sorted({k for k in base if mts[k].get("topic", b["source"]) != b["source"]}) + if foreign: + refuse(key, f"base measure(s) {foreign} live on another topic — cross-topic measures " + f"are scalar-only and are refused under a group_by") + continue + empty = m.get("empty") + if empty not in ("zero", "blank"): + refuse(key, "declares no `empty:` family (C1: zero for additive, blank for a ratio)") + continue + ftype = _FORMAT_TYPE.get(m.get("format")) + if ftype not in ("currency", "int", "pct"): + refuse(key, f"format {m.get('format')!r} has no grid measure type " + f"(aios_grid.MEASURE_FIELD_TYPES is currency|int|pct)") + continue + out.append({ + "key": key, + "label": m.get("label") or key, + "type": ftype, + "format": m.get("format"), + "empty": empty, + # The denominator whose zero blanks a ratio. None for an additive metric. + "guard": m.get("denominator") if m.get("agg") == "ratio" else None, + # ⛔⛔ TWO NORMALIZERS OF ONE WORD, and this is the seam between them + # ([[one-question-two-normalizers]]). A `pct` METRIC is a FRACTION internally + # (`margin_pct` = 0.5836); a `pct` GRID COLUMN is PERCENTAGE POINTS — the client + # renders it `num(v).toFixed(1) + "%"` (`customer-grid/cells.ts`), and the pool's own + # `yoy_pct` has always been on that scale (`core/periods.yoy_pct` multiplies by 100). + # Ship the fraction and a 58.4% margin prints as "0.6%": plausible, wrong, and nothing + # errors. `measure_filter.resolve_values` states the same rule for the customer path + # ("`input_scale` applied OUTWARD… the CELL shows points too") and this is that rule, + # derived from the metric's own `format` rather than a second hand-written list. + # ⚠ APPLIED ONCE, in `entity_measure_values`, so cells AND conditions are both in + # display units and the filter needs no inward scaling of the typed value. + "scale": 100.0 if m.get("format") == "pct" else 1.0, + # Cents for money (D-153: whole-dollar money cells are a shipped defect), one decimal + # for a percent — the convention `resolve_values` documents, minus its money rounding. + "round": 1 if m.get("format") == "pct" else (2 if ftype == "currency" else None), + "description": m.get("description") or "", + "topic": b["source"], + "dim": b["dim"], + }) + return out, refused + + +def entity_measure_values(topic, keys, date_from=None, date_to=None, team_id=None, + exclude_services=False, offer=None): + """`{group key: {measure key: value}}` for an entity topic's lookback columns. + + ONE grouped query for all `keys` together — they share a topic, a scope and a window, so + asking them separately would be N scans of the same rows (measured: the six ship-first + product metrics cost 0.22 s together against the mirror). + + ⭐ THE GROUP KEY IS THE SOURCE DIM'S VALUE, NOT A pid. This layer does not know how a grid + hashes its identity (`product_data.sku_pid` is a CRC32; an agent row's id is the partner id), + so the CALLER maps. Keeping the mapping at the caller is what lets one engine serve both. + + ⛔⛔ `exclude_services` DEFAULTS TO **FALSE** HERE, AND THAT IS THE OPPOSITE OF A RANKING + QUERY — measured, after the agent reconciliation went red on it. The service filter exists so + Delivery Charges do not top a "best SKUs" list; on an ENTITY LOOKBACK COLUMN the row IS the + entity, and dropping part of its activity makes the cell a different number from the one every + other surface shows for the same subject. Concretely: with services excluded, per-agent revenue + came in **2.87% under** an independent Odoo order-header aggregate and FOUR of nine agents + missed their own figure by 4-11%; with them included it ties. A service product on the product + grid has the same problem in reverse — its own row would read $0 while it genuinely sold. + ⚠ So a caller that wants a RANKING must pass `exclude_services=True` deliberately. The default + is the one that makes a per-row cell true. + + ⛔ THE EMPTY-WINDOW RULE IS APPLIED HERE, not left to a renderer, because it is the difference + between a true and a false cell (contract C1): + * a group that is ABSENT gets nothing back — the caller fills `zero`-family keys with 0 and + leaves `blank`-family keys out, which is what `_entity_zero_fill` does; + * a group that is PRESENT but whose ratio denominator is 0 has its ratio DROPPED here. + `_post_compute` answers 0.0 for `num/0` and on a grid that prints "0.0%" — a margin + nobody measured, under a row the user has no reason to doubt. + """ + offer = offer if offer is not None else entity_measures(topic) + by_key = {m["key"]: m for m in offer} + want = [k for k in keys if k in by_key] + if not want: + return {} + # ⭐ ONE QUERY PER SOURCE TOPIC (W37-T13). Keys from `sales_lines` and keys from `stock_moves` + # are different tables joined by different dims, so they cannot ride one scan — but every key + # WITHIN a source still does, which is the whole reason the six product metrics cost one query. + groups = {} + for k in want: + groups.setdefault((by_key[k]["topic"], by_key[k]["dim"]), []).append(k) + if len(groups) > 1: + merged = {} + for (src_, dim_), ks in groups.items(): + for gk, cell in entity_measure_values( + topic, ks, date_from=date_from, date_to=date_to, team_id=team_id, + exclude_services=exclude_services, offer=offer).items(): + merged.setdefault(gk, {}).update(cell) + return merged + (src, dim), want = next(iter(groups.items())) + res = store_query(src, want, group_by=[dim], date_from=date_from, date_to=date_to, + team_id=team_id, exclude_services=exclude_services, limit=MAX_GROUPS) + # ⛔ A TRUNCATED GROUP SET IS A WRONG ANSWER PER ROW, not a short list — the same refusal + # `rollup_sql.group_values` makes, for the same reason. Never write cells from one. + if res.get("truncated"): + raise _EntityMeasureError( + f"{topic}: {src} grouped by {dim} exceeded {MAX_GROUPS} groups — refusing to write " + f"cells from a truncated result") + # `store_query` emits the bare dim name when the dim has no `name_col`, `_id` when it + # does. Both shapes are live in the model, so read whichever arrived. + d = (_model()["topics"][src]["store"]["dims"] or {})[dim] + kcol = f"{dim}_id" if d.get("name_col") else dim + out = {} + for row in res["rows"]: + gk = row.get(kcol) + if gk is None: + continue # an unattributed group keys nothing; never key on None + cell = {} + for k in want: + m = by_key[k] + if m["guard"] is not None and not row.get(m["guard"]): + continue # ratio over a zero denominator -> blank, never 0.0 + v = row.get(k) + if v is None: + continue + # ⛔ THE SCALE BOUNDARY, crossed exactly once — see `scale` in `_entity_offer`. + v = v * m["scale"] if m["scale"] != 1.0 else v + if m["round"] is not None and isinstance(v, float): + v = round(v, m["round"]) + cell[k] = v + out[gk] = cell + return out + + +def entity_zero_fill(cell, keys, offer): + """Complete one row's cells under C1's empty-window rule -> the dict to hand `derived`. + + Additive keys land as a real `0` (it sold nothing, and that is a measurement); ratio keys stay + ABSENT so the grid paints an empty cell. Called for EVERY row, including the ones with no + group at all — which is 72% of the product catalogue in a 90-day window, and the reason this + rule is a contract rather than a default. + """ + by_key = {m["key"]: m for m in offer} + out = dict(cell or {}) + for k in keys: + m = by_key.get(k) + if not m or k in out: + continue + if m["empty"] == "zero": + out[k] = 0 + return out + + +def entity_measure_leaves(nodes, admitted): + """Every MEASURE leaf in a filter tree whose `colId` is in `admitted` — the entity twin of + `measure_filter.collect`, which is bound to the CUSTOMER vocabulary by its own `ADMITTED`.""" + found = [] + for n in nodes or []: + if not isinstance(n, dict): + continue + if isinstance(n.get("children"), list): + found.extend(entity_measure_leaves(n["children"], admitted)) + elif n.get("colId") in admitted: + found.append(n) + return found + + +def entity_measure_sets(topic, rules, today, team_id=None, keys_by_id=None, offer=None, + exclude_services=False): + """`{rule id: {group key, …}}` — a measure CONDITION answered as a set, per entity row. + + ⛔⛔ WHY THIS EXISTS AT ALL, because it is not obvious from T10's ticket: serving a non-empty + `measures` list does not only offer the Metric COLUMN, it offers the measure CONDITION too — + the client feeds the same array to its filter builder. A condition whose id never comes back + in `measureSets` is rendered PENDING ("Calculating…") and matches NOTHING, forever. So the + offer and this resolver are one feature; shipping the first alone converts a working filter + panel into a permanent spinner. + + ⭐ THE POPULATION IS THE WHOLE POOL FOR AN ADDITIVE MEASURE, zero-filled — which is the point. + 71.5% of the product catalogue has no group in a 90-day window, so `Revenue < 100` is a + question ABOUT those rows. Dropping them (the shape a naive GROUP BY gives you) would return + exactly the opposite set from the one the user asked for. + ⚠ A `blank`-family measure (a ratio) is NOT zero-filled and a row without one is simply not + comparable — "GM % below 40%" must not match a SKU that sold nothing, because it has no + margin percentage at all. + + ⚠ A rule that is INCOMPLETE or unanswerable is left OUT of the answer rather than resolved to + something — `measure_filter.rule_complete`'s reasoning, unchanged: `to_num(None)` is 0, so a + half-typed `Revenue > …` would silently become `Revenue > 0` under a confident count. + """ + from harness import measure_filter as _mf # pure helpers only: percentile / _CMP / STATS + from harness import windows as _wn + + offer = offer if offer is not None else entity_measures(topic) + by_key = {m["key"]: m for m in offer} + if not by_key: + return {} + cache = {} + + def values_for(mkey, window): + """`{group key: value}` for one (measure, window), memoised within this call.""" + w = _wn.normalize(window) + sig = (mkey, None if w is None else tuple(sorted(w.items()))) + if sig not in cache: + rng = _wn.resolve(window, today) + if rng is None: + cache[sig] = None # never widen to all time — see resolve() + else: + vals = entity_measure_values(topic, [mkey], date_from=rng[0], date_to=rng[1], + team_id=team_id, exclude_services=exclude_services, + offer=offer) + cache[sig] = {gk: c[mkey] for gk, c in vals.items() if mkey in c} + return cache[sig] + + out = {} + for rule in rules or []: + rid = str(rule.get("id") or "") + if not rid or not _mf.rule_complete(rule): + continue + mkey = rule.get("colId") + m = by_key.get(mkey) + op = rule.get("op") + if not m or op not in _mf._CMP: + continue + left = values_for(mkey, rule.get("window")) + if left is None: + continue + pool = list(keys_by_id or left) + zero = m["empty"] == "zero" + rhs = rule.get("rhs") + try: + if isinstance(rhs, dict) and rhs.get("kind") == "measure": + rk = rhs.get("colId") + if rk not in by_key: + continue + right = values_for(rk, rhs.get("window")) + if right is None: + continue + rzero = by_key[rk]["empty"] == "zero" + hit = set() + for gk in pool: + a, b = left.get(gk), right.get(gk) + if a is None: + if not zero: + continue # a ratio nobody has is not comparable + a = 0 + if b is None: + if not rzero: + continue + b = 0 + if _mf._CMP[op](a, b): + hit.add(gk) + elif isinstance(rhs, dict) and rhs.get("kind") == "stat": + stat = rhs.get("stat") + if stat not in _mf.STATS or op not in _mf.PAIR_OPS: + continue + # ⚠ THE POPULATION IS THE ROWS THAT HAVE THIS MEASURE AT ALL — the owner's ruling + # of 2026-07-27, carried over verbatim in effect: with 71.5% of this catalogue at + # zero, including them would drag the 25th percentile to exactly 0.0 and make + # "below the bottom quartile" match nobody. + population = {gk: v for gk, v in left.items() + if (keys_by_id is None or gk in set(pool)) and v} + cut = _mf.percentile(list(population.values()), stat) + if cut is None: + hit = set() + else: + hit = {gk for gk, v in population.items() if _mf._CMP[op](v, cut)} + else: + target = float(str(rule.get("value")).strip()) + hit = set() + for gk in pool: + v = left.get(gk) + if v is None: + if not zero: + continue + v = 0 + if _mf._CMP[op](v, target): + hit.add(gk) + except (TypeError, ValueError): + continue # a non-numeric value is not a question + out[rid] = hit + return out + + +def store_parity(date_from=None, date_to=None): + """THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope.""" + import core.periods as P + if not date_from: + date_from, date_to = P.ytd() + out = [] + for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced", + "revenue_invoiced", "orders_invoiced"): # wave 21 R2 — both filtered paths + m = _model()["metrics"][key] + # ⚠ A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a + # team is asked of a topic with no team_col, and the live builder refuses too — so + # looping the BUs here would fail the gate for a topic that is correct. Parity for a + # company-level number means company-level parity; pretending otherwise would either + # crash or, worse, compare two numbers that silently ignored the team. + _store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {} + scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,) + for tid in scopes: + live = metric(key, date_from, date_to, tid) + res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid) + ours = res["rows"][0][key] if res["rows"] else 0 + gap = abs((ours or 0) - (live or 0)) + out.append({"check": f"store.{key} == live.{key} (team={tid})", + "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live}) + # The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the + # agent's FIRST-agent book — a fully independent path (live partner read + line-domain sum) + # against the store's res_partner.agent_id attribution. Largest book = the sharpest check. + recs = O.search_read("res.partner", [("active", "in", [True, False]), + ("agent_ids", "!=", False)], ["agent_ids"], limit=100000) + books = {} + for r in recs: + a = (r.get("agent_ids") or [None])[0] + if a: + books.setdefault(a, []).append(r["id"]) + if books: + top = max(books, key=lambda k: len(books[k])) + live = O.sum_field("sale.order.line", + O.sale_line_domain(date_from, date_to, partner_ids=books[top]), + "price_subtotal") + res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to, + filters={"agent": [top]}) + ours = res["rows"][0]["revenue"] if res["rows"] else 0 + gap = abs((ours or 0) - (live or 0)) + out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, " + f"{len(books[top])} customers)", + "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live}) + return out diff --git a/platform/model/metrics/sales.yml b/platform/model/metrics/sales.yml index c15b59507c773574ebd1ce8cd3b2835e3cc6e690..37cf3b9e2af70063779b9d62515ef0543bf89472 100644 --- a/platform/model/metrics/sales.yml +++ b/platform/model/metrics/sales.yml @@ -1,108 +1,149 @@ -# Metrics: sales — the core wholesale metrics, defined ONCE (OM-0). Every surface (pages, the -# metric dictionary, the Analyst, MCP) resolves these by key through harness/semantic.py. -# Fields per metric: -# key/label/description — identity + the human definition (visibility = trust) -# agg + field — sum | count_distinct over the topic's entity -# agg: ratio — numerator/denominator are metric KEYS (resolved recursively) -# agg: derived + expr — arithmetic over metric keys (safe parser; +,-,*,/ and parens only) -# format — usd | int | pct (rendering hint for surfaces) -# ai_context — what a small model must know to use the metric correctly -# validate — the INDEPENDENT Odoo cross-check contract (named method implemented in -# harness/semantic.py _VALIDATORS; a metric that can't tie out says so) -topic: sales_lines -metrics: - - key: revenue - label: Revenue - agg: sum - field: price_subtotal - format: usd - description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)." - ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)." - validate: - method: order_level_revenue - note: "Σ line price_subtotal must equal Σ parent-order amount_untaxed under the same scope, to the cent (line vs order basis — the built-in cross-check)." - - - key: units - label: Units sold - agg: sum - field: product_uom_qty - format: int - description: "Total quantity across confirmed wholesale order lines." - ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first." - - - key: margin - label: Gross margin $ - agg: sum - field: margin - format: usd - description: "Line revenue minus line cost (Odoo Margin module), summed." - ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost — never sum it as a total." - - - key: cogs - label: COGS - agg: derived - expr: "revenue - margin" - format: usd - description: "Cost of goods sold, derived: revenue minus gross margin." - ai_context: "Derived, not pulled — Odoo carries cost on lines as margin; COGS is the difference." - - - key: margin_pct - label: Gross margin % - agg: ratio - numerator: margin - denominator: revenue - format: pct - description: "Gross margin as a share of revenue." - ai_context: "Guarded against zero revenue (returns 0). Compare across BUs/categories at the same scope only." - - - key: orders - label: Orders - topic: sales_orders - agg: count - format: int - description: "Confirmed wholesale orders in the window (order-header count — what the scorecards show)." - ai_context: "Order-level count; slightly higher than distinct-orders-from-lines because a few confirmed orders carry zero lines (a data-health artifact the reconciliation contract counts exactly)." - validate: - method: order_count - note: "Order-level count must equal distinct line-parent orders + line-less orders, exactly (surfaces empty orders instead of hiding them in a tolerance)." - - - key: customers - label: Active customers - agg: count_distinct - field: order_partner_id - format: int - description: "Distinct customers with at least one confirmed wholesale order line in the window." - ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id." - - - key: aov - label: Average order value - agg: ratio - numerator: revenue - denominator: orders - format: usd - description: "Revenue per distinct order." - ai_context: "Ratio of two registered metrics at identical scope; never average per-order averages." - - # ⭐ Wave 21 R2 — the FULLY-INVOICED basis, as separate metrics (owner ruling: picker entries, - # not a basis dropdown). Same topics, same scope, ONE predicate narrower: the order's Odoo - # invoice_status = 'invoiced'. `store_filter_sql` filters the store path (sum/count CASE); - # `live_domain` filters the live path — BOTH or store_parity compares two different questions. - - key: revenue_invoiced - label: Sales — fully invoiced - agg: sum - field: price_subtotal - store_filter_sql: "o.invoice_status = 'invoiced'" - live_domain: [["order_id.invoice_status", "=", "invoiced"]] - format: usd - description: "Untaxed revenue of confirmed wholesale order lines whose parent order Odoo marks fully invoiced (invoice_status = invoiced)." - ai_context: "Same scope as revenue, narrowed by the ORDER-level fully-invoiced flag. This is NOT posted-invoice-line revenue: a partially invoiced order is excluded entirely until Odoo flips the flag. Reconciled store-vs-live per BU by store_parity." - - - key: orders_invoiced - label: Orders — fully invoiced - topic: sales_orders - agg: count - store_filter_sql: "o.invoice_status = 'invoiced'" - live_domain: [["invoice_status", "=", "invoiced"]] - format: int - description: "Confirmed wholesale orders Odoo marks fully invoiced (invoice_status = invoiced)." - ai_context: "Order-header count under the orders scope plus the fully-invoiced flag; partially invoiced and not-yet-invoiced orders are excluded. Reconciled store-vs-live per BU by store_parity." +# Metrics: sales — the core wholesale metrics, defined ONCE (OM-0). Every surface (pages, the +# metric dictionary, the Analyst, MCP) resolves these by key through harness/semantic.py. +# Fields per metric: +# key/label/description — identity + the human definition (visibility = trust) +# agg + field — sum | count_distinct over the topic's entity +# agg: ratio — numerator/denominator are metric KEYS (resolved recursively) +# agg: derived + expr — arithmetic over metric keys (safe parser; +,-,*,/ and parens only) +# format — usd | int | pct (rendering hint for surfaces) +# ai_context — what a small model must know to use the metric correctly +# validate — the INDEPENDENT Odoo cross-check contract (named method implemented in +# harness/semantic.py _VALIDATORS; a metric that can't tie out says so) +# empty — W37 C1's EMPTY-WINDOW FAMILY. See below; a metric an ENTITY topic +# offers as a lookback column MUST declare one. +# +# ⛔⛔ `empty:` — WHAT A ROW WITH NO ACTIVITY IN THE WINDOW RENDERS AS (wave 37, contract C1). +# Measured and load-bearing: only 1,646 of 5,836 active products sold in the last 90 days, so a +# per-SKU lookback metric has NO GROUP for 72% of the catalogue. Get the default wrong and every +# product grid reads as broken. Two values, and the difference is whether the blank cell would be +# a TRUE STATEMENT: +# empty: zero ADDITIVE — units, revenue, margin $, COGS. "It sold nothing" is a real +# measurement, so 0 is the honest cell and a blank would hide a fact. +# empty: blank RATIO / DERIVED-FROM-A-RATIO — GM %, ASP. A 0% margin on zero sales is a +# FALSE statement, not a missing one. ⛔ AND THE GUARD IS THE DENOMINATOR, NOT +# THE MISSING GROUP: `semantic._post_compute` returns 0.0 for `num/0`, so a SKU +# that DID sell at $0 would print "0.0%" with a group behind it. The resolver +# blanks on a zero denominator, which is the only reading that is never a lie. +# ⚠ A metric with no `empty:` is REFUSED by `entity_measures()` rather than defaulted — a +# defaulted family is a guess about truth, and C1 says a metric that cannot say which family it +# is in does not ship. +topic: sales_lines +metrics: + - key: revenue + label: Revenue + agg: sum + field: price_subtotal + format: usd + empty: zero + description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)." + ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)." + validate: + method: order_level_revenue + note: "Σ line price_subtotal must equal Σ parent-order amount_untaxed under the same scope, to the cent (line vs order basis — the built-in cross-check)." + + - key: units + label: Units sold + agg: sum + field: product_uom_qty + format: int + empty: zero + description: "Total quantity across confirmed wholesale order lines." + ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first." + + - key: margin + label: Gross margin $ + agg: sum + field: margin + format: usd + empty: zero + description: "Line revenue minus line cost (Odoo Margin module), summed." + ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost — never sum it as a total." + + - key: cogs + label: COGS + agg: derived + expr: "revenue - margin" + format: usd + empty: zero + description: "Cost of goods sold, derived: revenue minus gross margin." + ai_context: "Derived, not pulled — Odoo carries cost on lines as margin; COGS is the difference." + + - key: margin_pct + label: Gross margin % + agg: ratio + numerator: margin + denominator: revenue + format: pct + empty: blank + description: "Gross margin as a share of revenue." + ai_context: "Compare across BUs/categories at the same scope only. ⚠ TWO ZERO-REVENUE BEHAVIOURS, deliberately: the scalar/analyst path returns 0 (semantic._post_compute's guard), while an ENTITY LOOKBACK COLUMN renders BLANK (empty: blank) — on a grid, a printed 0.0% beside 5,836 products would assert a margin nobody measured." + + # ⭐ W37-T10 — ASP, the fifth SHIP-FIRST per-SKU metric (proto/P3). Blended $18.76 over 1,646 + # SKUs in the 90 days to 2026-08-19. A RATIO of two same-topic base measures, so it survives a + # GROUPED store_query (only a CROSS-TOPIC component is refused — that is what stops `aov`, + # whose denominator `orders` lives on sales_orders). + # ⚠ UNITS ARE AS-ORDERED, mixed UoM. `units` says so and this inherits it: a SKU sold in cases + # and in singles has an ASP blended across both, which is the true average selling price of a + # line unit and NOT a per-piece price. + - key: asp + label: Average selling price + agg: ratio + numerator: revenue + denominator: units + format: usd + empty: blank + description: "Revenue per unit sold — the blended average selling price." + ai_context: "revenue ÷ units at identical scope. Mixed units of measure are summed as ordered quantity, so this is per ordered unit, not per piece. Blank when nothing sold: an ASP of $0 on zero units is a false statement, not a missing one." + + - key: orders + label: Orders + topic: sales_orders + agg: count + format: int + description: "Confirmed wholesale orders in the window (order-header count — what the scorecards show)." + ai_context: "Order-level count; slightly higher than distinct-orders-from-lines because a few confirmed orders carry zero lines (a data-health artifact the reconciliation contract counts exactly)." + validate: + method: order_count + note: "Order-level count must equal distinct line-parent orders + line-less orders, exactly (surfaces empty orders instead of hiding them in a tolerance)." + + - key: customers + label: Active customers + agg: count_distinct + field: order_partner_id + format: int + empty: zero + description: "Distinct customers with at least one confirmed wholesale order line in the window." + ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id." + + - key: aov + label: Average order value + agg: ratio + numerator: revenue + denominator: orders + format: usd + description: "Revenue per distinct order." + ai_context: "Ratio of two registered metrics at identical scope; never average per-order averages." + + # ⭐ Wave 21 R2 — the FULLY-INVOICED basis, as separate metrics (owner ruling: picker entries, + # not a basis dropdown). Same topics, same scope, ONE predicate narrower: the order's Odoo + # invoice_status = 'invoiced'. `store_filter_sql` filters the store path (sum/count CASE); + # `live_domain` filters the live path — BOTH or store_parity compares two different questions. + - key: revenue_invoiced + label: Sales — fully invoiced + agg: sum + field: price_subtotal + store_filter_sql: "o.invoice_status = 'invoiced'" + live_domain: [["order_id.invoice_status", "=", "invoiced"]] + format: usd + description: "Untaxed revenue of confirmed wholesale order lines whose parent order Odoo marks fully invoiced (invoice_status = invoiced)." + ai_context: "Same scope as revenue, narrowed by the ORDER-level fully-invoiced flag. This is NOT posted-invoice-line revenue: a partially invoiced order is excluded entirely until Odoo flips the flag. Reconciled store-vs-live per BU by store_parity." + + - key: orders_invoiced + label: Orders — fully invoiced + topic: sales_orders + agg: count + store_filter_sql: "o.invoice_status = 'invoiced'" + live_domain: [["invoice_status", "=", "invoiced"]] + format: int + description: "Confirmed wholesale orders Odoo marks fully invoiced (invoice_status = invoiced)." + ai_context: "Order-header count under the orders scope plus the fully-invoiced flag; partially invoiced and not-yet-invoiced orders are excluded. Reconciled store-vs-live per BU by store_parity." diff --git a/platform/model/metrics/stock.yml b/platform/model/metrics/stock.yml new file mode 100644 index 0000000000000000000000000000000000000000..51a10c56143a9b1fed87714e3e7b1d84d9de2e5f --- /dev/null +++ b/platform/model/metrics/stock.yml @@ -0,0 +1,57 @@ +# Metrics: stock — physical movement (W37-T13, owner item 4 / ruling R1). +# +# ⛔⛔ THE DIRECTION LIVES IN `store_filter_sql`, WHICH IS WHY THESE ARE TWO METRICS AND NOT ONE +# METRIC WITH A PARAMETER. The model's filtered-measure support compiles `store_filter_sql` into +# `sum(CASE WHEN THEN ELSE 0 END)`, so IN and OUT are the SAME sum over the +# SAME rows under two different predicates — one scan answers both, and neither can drift from the +# other's scope because they share every other clause. +# +# ⛔ EACH PREDICATE IS TWO-SIDED, and that is the ticket's headline trap (`proto/P1-stock-moves.md`): +# `internal -> internal` is **58% of all moves**. A one-sided test (`destination is internal`) counts +# every internal transfer as an arrival AND a one-sided `source is internal` counts it as a +# departure — both columns roughly double, nothing errors, and the totals stay internally +# consistent with each other, which is what makes it survive review. +# +# ⚠ `usage` COMES FROM `stock_location`, NOT from the move. `location_id` mirrors as an id plus a +# NAME; there is no usage on the move row, so `stock_location` is joined in the topic precisely to +# make this predicate expressible. If that join is ever dropped, these two metrics do not go red — +# they go NULL-predicate, which SQL treats as false, and both columns silently read ZERO. +# +# ⚠ ADJUSTMENTS AND SCRAP ARE INCLUDED, DELIBERATELY, AND THE ALTERNATIVE WAS MEASURED. They are +# ~30% of IN and ~20% of OUT, and the two obvious exclusions are NOT equivalent — the 5,320-unit +# gap between them is entirely `Virtual Locations/Scrap`. A physical arrival is an arrival however +# it was booked, so the honest default is to count it and let the `source`/`destination` dims +# separate it. Excluding it silently would be a different number under the same label. +topic: stock_moves +metrics: + - key: stock_in + label: Stock moved in + agg: sum + field: quantity_done + store_filter_sql: "sd.usage = 'internal' AND COALESCE(sl.usage, '') <> 'internal'" + format: int + empty: zero + description: "Units that ARRIVED at an internal location from outside it, over the window." + ai_context: "Receipts, returns inward and inventory adjustments upward. Two-sided by construction: an internal-to-internal transfer is NOT an arrival. Quantities are in each product's own unit, so never sum this across SKUs." + + - key: stock_out + label: Stock moved out + agg: sum + field: quantity_done + store_filter_sql: "sl.usage = 'internal' AND COALESCE(sd.usage, '') <> 'internal'" + format: int + empty: zero + description: "Units that LEFT an internal location for outside it, over the window." + ai_context: "Deliveries, scrap and adjustments downward. Two-sided by construction: an internal-to-internal transfer is NOT a departure. Quantities are in each product's own unit, so never sum this across SKUs." + + # ⚠ NET IS DERIVED, never a third sum. Deriving it guarantees `net = in - out` exactly, where a + # separately-summed net could disagree with its own two components under a filter and nothing + # would say which was right. + - key: stock_net + label: Stock moved net + agg: derived + expr: "stock_in - stock_out" + format: int + empty: zero + description: "Arrivals minus departures over the window — the period change in units held." + ai_context: "Derived from stock_in and stock_out, so it always reconciles to them. It is a MOVEMENT figure, not the on-hand balance: the balance is `product_data.on_hand` from stock.quant." diff --git a/platform/model/topics/odoo_agents.yml b/platform/model/topics/odoo_agents.yml index 3353f817c0d4a5fb661b3ef4f63c0e32ecf3e7c0..3e6664d76dae2ed91517965301ea6e128968ebd8 100644 --- a/platform/model/topics/odoo_agents.yml +++ b/platform/model/topics/odoo_agents.yml @@ -21,6 +21,38 @@ store: # NO date_col — a registry is not a dated event stream. Stated rather than # omitted, so its absence reads as a fact and not as an unfinished file. +# ⭐⭐ W37-T12 / owner item 4 (R1) — THE LOOKBACK-MEASURE BINDING, the agent half of +# *"It should apply to Odoo agents database as well."* Same contract as +# `odoo_products.yml`: a FACT topic, and the dim of that topic which carries THIS +# entity's identity. +# +# ⛔⛔ WHICH AGENT SOURCE, STATED — because there are THREE in this Odoo and they name +# DIFFERENT PEOPLE (`proto/P3-metric-catalog.md`: 9 agents carry route-1 revenue, 12 +# carry commission lines, 17 carry the flag). This binds ROUTE 1, the CUSTOMER-MASTER +# BOOK: `sales_lines.agent` is `rp.agent_id`, the customer's assigned agent, so every +# order of that customer counts toward their agent. That is "whose book is this" and it +# is the right question for a column on the AGENT REGISTRY. +# ⚠ It is NOT commission. `account.invoice.line.agent` (topic `commission_lines`) is the +# per-INVOICE-LINE credited agent and answers a different question; `sales_lines.yml`'s +# own `agent` dim comment carries the full disagreement and says never to "fix" one by +# reading the other. +# +# ⭐ THE JOIN NEEDS NO NEW DIM, unlike the product side. `sales_lines.agent` declares a +# `name_col`, so `store_query` emits `agent_id` — which IS `res.partner.id`, which IS +# this grid's `odoo_id` identity. Product needed `product_code` because its grid keys on +# `default_code`; this one already keys on the same integer the fact topic groups by. +measures: + source: sales_lines + dim: agent + keys: [revenue, units, margin, cogs, margin_pct, asp, customers] + not_yet: + - key: orders / aov + cause: "`orders` lives on topic `sales_orders`, so under a `group_by` it is a CROSS-TOPIC measure and `semantic.store_query` refuses it (scalar-only). `aov` inherits the refusal through its denominator" + fix: "add an ORDER-COUNT metric to `sales_lines` itself — `count_distinct` over `l.order_id` — which answers the same question at this grain in one pass" + - key: commission_amount + cause: "the commission basis lives on `account.invoice.line.agent` (topic `commission_lines`), a different topic AND a different grain; `invoice_lines` deduplicates to at most one agent per line" + fix: "bind a SECOND measures block per source once the contract allows more than one, and label the columns so the two routes can never be read as the same number" + # key / label / type / kind, derived from the grid contract. `kind` says where the # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed # from another topic; a `link` points at another database. diff --git a/platform/model/topics/odoo_products.yml b/platform/model/topics/odoo_products.yml index 05d5ddb16decb15789268c207a6d9b27397a8cea..581e0a014d2b0440b589fe5df19debbe2276a299 100644 --- a/platform/model/topics/odoo_products.yml +++ b/platform/model/topics/odoo_products.yml @@ -26,6 +26,57 @@ store: category: {label: "Category"} supplier: {label: "Supplier"} +# ⭐⭐ W37-T10 / owner item 4 (ruling R1) — THE LOOKBACK-MEASURE BINDING. Owner: *"Odoo products +# should have a lookbsck metrics like sales etc."* This block is what turns `measures: []` on the +# product workspace into a real catalogue, and it is DECLARED HERE rather than hand-written in a +# route for the reason `_rollup_source_offer` gives about itself: a second list is a second +# definition of the same fact, and the two drift the day somebody adds a metric. +# +# ⛔ THE JOIN KEY IS THE TRAP CONTRACT C1 NAMES, and this is where it is answered. `source` is a +# FACT topic; `dim` is the dim of THAT topic which carries THIS entity's identity. `sales_lines` +# has two candidates and only one is right: `product` groups by Odoo's `product_id`, while this +# database's identity (see `scope.identity` above) is `default_code`. Binding to `product` would +# key every cell on a number no grid row carries — a column of nulls that reads as "never sold". +# +# ⚠ A SINGLE-TOPIC GROUPED QUERY, which is why `semantic.store_query`'s cross-topic refusal does +# not bite: `revenue` lives ON `sales_lines` and we group `sales_lines` by its OWN dim. The same +# refusal is real and DOES bite `aov`/`orders` (denominator `orders` lives on `sales_orders`), so +# those two are absent from `keys` below and `entity_measures()` re-proves that rather than +# trusting this list. +# ⭐ A LIST SINCE W37-T13: an entity's columns legitimately come from more than ONE fact topic. +# Sales movement is `sales_lines`; physical movement is `stock_moves`. Same grid, same join key +# (`product_code`), different tables — forcing stock into the sales topic would have defined a +# metric against rows it does not have. +measures: +- source: sales_lines + dim: product_code + # ⭐ The SHIP-FIRST six of `proto/P3-metric-catalog.md`, and they cost ONE grouped query + # together (measured 0.22 s over 1,670 groups against the mirror; 2.55 s live). + keys: [units, revenue, margin, cogs, margin_pct, asp] + # ⛔ REPORTED, NOT SILENTLY DROPPED (standing rule 1's second sentence, applied to a catalogue). + # Each of these is a real metric `proto/P3-metric-catalog.md` measured and this wave does not + # ship, with the CAUSE and the fix — so the next session extends the list instead of + # re-measuring, and nobody reads the six as "all Odoo can answer". + not_yet: + - key: days_since_last_sale + cause: "needs `agg: max` over a DATE, which `semantic._measure_sql` does not implement (sum | count | count_distinct only), and a date-typed measure could not render anyway: `aios_grid.MEASURE_FIELD_TYPES` is {currency, int, pct}" + fix: "add `agg: max` to `_measure_sql` and express the metric as an INT — `date_diff('day', max(date), today)` — so it renders as a number of days rather than a date. Measured live at 4.61 s over 3,348 SKUs" + - key: distinct_orders + cause: "`__count` on the live path is LINE count, not ORDER count (12,707 lines vs 12,645 distinct SKU-order pairs); the honest figure needs a 2-level groupby measured at 10.87 s live" + fix: "on the STORE path this is `count(DISTINCT l.order_id)` in one pass — add it as a `count_distinct` metric on `sales_lines` with field `order_id`" + +# ⭐⭐ W37-T13 — PHYSICAL MOVEMENT. `stock.move` and `stock_location` are mirrored now (the +# `not_yet` entry that used to sit here said "not mirrored, so there is no store binding to +# group"; that is what changed). The direction rule is TWO-SIDED and lives in the metrics' +# `store_filter_sql` — see `model/metrics/stock.yml`, which carries the 58% trap in full. +- source: stock_moves + dim: product_code + keys: [stock_in, stock_out, stock_net] + not_yet: + - key: stock_in_excl_adjustments + cause: "adjustments and scrap are ~30% of IN / ~20% of OUT and the two obvious exclusions are NOT equivalent - the 5,320-unit gap between them is entirely `Virtual Locations/Scrap`, so the choice is a business ruling rather than a filter" + fix: "ask the owner which exclusion they mean, then add it as a SEPARATE metric with `store_filter_sql` narrowed on `sl.usage`/`sd.usage` - never by changing what `stock_in` means" + # key / label / type / kind, derived from the grid contract. `kind` says where the # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed # from another topic; a `link` points at another database. diff --git a/platform/model/topics/sales_lines.yml b/platform/model/topics/sales_lines.yml index e0b67f7b7d04b4408e62fc5fd8fde8793bafbf5b..5552441627296279752319bedcff6fef7f0c536e 100644 --- a/platform/model/topics/sales_lines.yml +++ b/platform/model/topics/sales_lines.yml @@ -42,6 +42,19 @@ store: dims: order_partner: {col: "l.order_partner_id", name_col: "l.order_partner_name", label: "Customer"} product: {col: "l.product_id", name_col: "l.product_name", label: "Product"} + # ⭐⭐ W37-T10 — THE SKU-CODE DIM, added because a per-SKU metric has to group by the key the + # PRODUCT GRID actually carries. `product` above groups by Odoo's `product_id`; the + # `odoo_products` database's identity is `default_code` (its own topic file says so), and a + # metric keyed on the wrong one is a column of nulls that reads as "this never sold". + # ⚠ `COALESCE(NULLIF(...))` REPRODUCES THE GRID'S IDENTITY EXACTLY, including the codeless + # fallback: `modules/product_data.py` keys an uncoded active product as `pid:`, so + # this expression has to as well or those rows silently lose their metrics. + # ⭐ It also MERGES a re-SKUed pair: an archived record and the active one sharing a code are + # one product to the grid, and grouping by `product_id` would split them into two. + # ⚠ NO `name_col` — the code IS its own label, so `store_query` emits the bare `product_code` + # (the value-keyed shape `rollup_sql.group_values` handles beside `payment_state`). + product_code: {col: "COALESCE(NULLIF(pp.default_code, ''), 'pid:' || CAST(pp.id AS VARCHAR))", + label: "SKU code"} # team names via CASE (crm_team is not synced; ids are baked tenant scope like the team filter) team: {col: "o.team_id", name_col: "CASE o.team_id WHEN 5 THEN 'Fisch' WHEN 6 THEN 'Royal' ELSE CAST(o.team_id AS VARCHAR) END", diff --git a/platform/model/topics/stock_moves.yml b/platform/model/topics/stock_moves.yml new file mode 100644 index 0000000000000000000000000000000000000000..14f27d9d82220791db7dc95fb1595e0a6952e745 --- /dev/null +++ b/platform/model/topics/stock_moves.yml @@ -0,0 +1,72 @@ +# ⭐⭐ W37-T13 (owner: stock moved IN and OUT, per SKU, over a lookback window). +# +# A DATED EVENT STREAM at move grain — the counterpart to `odoo_products`, which is a +# catalogue and says so ("a catalogue has no date dimension; ask sales_lines for +# movement"). This is the topic that answers movement for stock, as `sales_lines` does +# for revenue. +# +# ⛔ EVERY CLAUSE HERE WAS MEASURED, in `proto/P1-stock-moves.md`, against live Odoo. +# Do not "simplify" one without re-reading it — three of them are traps that produce a +# plausible wrong number rather than an error. +key: stock_moves +label: Stock movement +entity: stock.move +domain_builder: ~ +scope: + state: "done moves ONLY — a draft/waiting/assigned move is an intention, not a movement" + direction: >- + DIRECTION IS A PROPERTY OF THE PAIR OF LOCATIONS, never of one. A move is IN when it + arrives at an internal location from a non-internal one, and OUT when it leaves an + internal location for a non-internal one. internal->internal is 58% of all moves and + is NEITHER; counting it as both is the defect a one-sided domain produces. + not_picking_code: >- + `picking_code` is NOT the discriminator. It is store=False, so a groupby faults, and + ('picking_code','=',False) returns 0 while 5,800 pickingless done moves exist - the + count that would warn you also reads 0. + scale: "236k rows over 1,095 days; cost is FLAT in window length (30d 2.1s .. 1095d 4.0s live)" + no_bu: >- + `stock.move` has no team_id. Stock is HQ-consolidated, like AR and Inventory - stated + as a fact rather than left to read as a missing filter. + +grain: "one row per done stock move; time-filterable by move date; NOT BU-filterable" + +store: + table: stock_move + alias: sm + # ⛔ BOTH LOCATION JOINS, and they are the whole of the direction rule. `sl` is where it + # came FROM, `sd` is where it went TO. Dropping either makes every direction metric + # silently one-sided. + join: >- + LEFT JOIN stock_location sl ON sl.id = sm.location_id + LEFT JOIN stock_location sd ON sd.id = sm.location_dest_id + LEFT JOIN product_product pp ON pp.id = sm.product_id + date_col: "sm.date" + # NO team_col — see scope.no_bu. A BU-scoped caller is REFUSED by `store_query` with a + # sentence rather than being served a company-wide number wearing a unit's label. + scope_sql: "sm.state = 'done'" + dims: + # ⭐ THE SAME CODE-KEYED SHAPE `sales_lines.product_code` uses, and for the same reason: + # the product grid's identity is `default_code`, with `pid:` for the codeless + # actives. A dim keyed on Odoo's `product_id` would key every cell on a value no grid + # row carries (contract C1's join-key trap). + product_code: {col: "COALESCE(NULLIF(pp.default_code, ''), 'pid:' || CAST(pp.id AS VARCHAR))", + label: "SKU code"} + product: {col: "sm.product_id", name_col: "pp.name", label: "Product"} + source: {col: "sm.location_id", name_col: "sl.complete_name", label: "From location"} + destination: {col: "sm.location_dest_id", name_col: "sd.complete_name", label: "To location"} + +ai_context: > + Physical stock movement at move grain, from Odoo `stock.move`, done moves only. + Use it for "what came in", "what went out", "what moved" over a window, per SKU or per location. + ⛔ DIRECTION IS A PAIR TEST, never a single location: `stock_in` counts moves ARRIVING at an + internal location from a non-internal one, `stock_out` counts moves LEAVING an internal location + for a non-internal one, and internal-to-internal transfers (58% of all moves) are in NEITHER. + Quantities are `quantity_done` — what actually moved — in the product's own unit of measure, so + units are NOT comparable across SKUs and must never be summed into one total across products. + ⛔ THIS TOPIC HAS NO BUSINESS UNIT. `stock.move` carries no team, so a question scoped to Fisch + or Royal cannot be answered here and must be refused rather than answered company-wide. + ⚠ Adjustments (`inventory` locations) and scrap are ~30% of IN and ~20% of OUT. `stock_in` and + `stock_out` INCLUDE them, because a physical arrival is an arrival however it was booked; use + the `source`/`destination` dims to separate them, and say which you did. + For SALES movement use sales_lines (revenue and units ordered); this topic is the warehouse, + and the two legitimately disagree because an order is not a shipment. diff --git a/platform/modules/agent.py b/platform/modules/agent.py index 9b9a019c369b108514d24439e3b19bc1c79ba352..05f8968b0d9136517a78364b2571efe9d980b169 100644 --- a/platform/modules/agent.py +++ b/platform/modules/agent.py @@ -1,277 +1,385 @@ -"""Agent module — per-agent (res.partner.agent_ids) analytics. - -An *agent* owns a **book** of customers (the same attribute the Customers module slices by). This -module reports that book the way Sales/Customers/SKU report the whole company: a period scorecard -with custom date windows (Today / WTD / Last week / MTD / QTD / YTD / any custom range), a sales -trend, returns, top SKUs (with profit/order) and the FULL customer list — INCLUDING inactive -accounts (no recent orders) so a rep sees who they've stopped selling to. - -Scope: reuses the Sales `order_domain` (Fisch+Royal, excluded accounts removed, state sale/done) -so numbers tie to every other module. Returns are consolidated (credit notes aren't BU-tagged); -everything else is BU-filterable via team_id. -""" -import sys -import datetime as dt -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -import core.odoo as O -import core.periods as P -import modules.sales as sales_mod -import modules.customers as cust_mod - - -def options(t=None, team_id=None): - """Agent names with book activity — for the page/drawer picker.""" - return cust_mod.agent_options(t, team_id) - - -def _book(name): - """frozenset of every partner id assigned to the agent (incl. inactive). None only for 'All'.""" - return cust_mod.agent_partner_ids(name) - - -def _rev(date_from, date_to, team_id, book): - return O.sum_field('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), - 'amount_untaxed') - - -def _orders(date_from, date_to, team_id, book): - return O.get_odoo().search_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book)) - - -def _custs(date_from, date_to, team_id, book): - return O.distinct_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), - 'partner_id') - - -# ------------------------------------------------------------ period scorecard (custom dating) -# Today / WTD / Last week / MTD / QTD / YTD — each carries its window so the UI can decompose it -# and so "what did this agent sell this/last week" is a click, not a date-math exercise. -_PERIODS = [('Today', 'today'), ('Week to date', 'wtd'), ('Last week', 'lwk'), - ('Month to date', 'mtd'), ('Quarter to date', 'qtd'), ('Year to date', 'ytd')] - - -def _window(key, t): - if key == 'today': - return P._d(t), P._d(t) - if key == 'lwk': # the full prior Mon–Sun week - f, _tt = P.wtd(t) - start = dt.date.fromisoformat(f) - dt.timedelta(days=7) - return start.isoformat(), (dt.date.fromisoformat(f) - dt.timedelta(days=1)).isoformat() - return {'wtd': P.wtd, 'mtd': P.mtd, 'qtd': P.qtd, 'ytd': P.ytd}[key](t) - - -def scorecard(name, t=None, team_id=None): - """Book revenue (+ YoY same-period), orders for Today / WTD / Last week / MTD / QTD / YTD.""" - t = t or P.today() - book = _book(name) - out = [] - for label, key in _PERIODS: - f, tt = _window(key, t) - wk = key in ('today', 'wtd', 'lwk') # weekday-align the short windows' LY compare - cf, ct = P.shift_year(f, tt, weeks=wk) - rev, rev_ly = _rev(f, tt, team_id, book), _rev(cf, ct, team_id, book) - out.append({'key': key, 'label': label, 'date_from': f, 'date_to': tt, 'cmp_from': cf, 'cmp_to': ct, - 'revenue': rev, 'revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly), - 'orders': _orders(f, tt, team_id, book)}) - return out - - -def headline(name, date_from, date_to, team_id=None): - """Book KPIs for an ARBITRARY window (custom dating): revenue + YoY (same window LY), orders, - active customers, AOV and returns $ / return rate.""" - book = _book(name) - cf, ct = P.shift_year(date_from, date_to, weeks=False) - rev, rev_ly = _rev(date_from, date_to, team_id, book), _rev(cf, ct, team_id, book) - orders = _orders(date_from, date_to, team_id, book) - ret = sales_mod._returns_amt(date_from, date_to, book) - return {'date_from': date_from, 'date_to': date_to, 'cmp_from': cf, 'cmp_to': ct, - 'revenue': rev, 'revenue_ly': rev_ly, - 'yoy_pct': P.yoy_pct(rev, rev_ly), 'orders': orders, - 'customers': _custs(date_from, date_to, team_id, book), 'aov': (rev / orders) if orders else 0.0, - 'returns': ret, 'return_rate_pct': (ret / rev * 100.0) if rev else 0.0} - - -# ------------------------------------------------------------ sales trend -def _book_monthly_rev(book, date_from, date_to, team_id=None): - g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), - ['amount_untaxed:sum'], ['date_order:month'], lazy=False) - out = {} - for r in g: - ym = ((r.get('__range') or {}).get('date_order:month') or {}).get('from', '')[:7] - if ym: - out[ym] = r.get('amount_untaxed') or 0.0 - return out - - -def monthly(name, n=13, t=None, team_id=None): - """Book sales per month vs the same month last year (one month-grouped query over 2 years).""" - t = t or P.today() - book = _book(name) - mrev = _book_monthly_rev(book, dt.date(t.year - 2, t.month, 1).isoformat(), t.isoformat(), team_id) - rows = [] - for ym, _s, _e in P.month_starts(n, t): - y, m = int(ym[:4]) - 1, int(ym[5:7]) - this, last = mrev.get(ym, 0.0), mrev.get(f'{y:04d}-{m:02d}', 0.0) - rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) - return rows - - -# ------------------------------------------------------------ full customer book (incl. inactive) -def customers(name, t=None, team_id=None): - """EVERY customer in the agent's book, including inactive accounts (no YTD/LY orders) — those - show $0 with status 'Inactive'/'Dormant'. Each row is clickable to the customer drawer and - carries recency so a rep can see who's gone quiet. Sorted by YTD revenue desc (inactive last).""" - t = t or P.today() - book = _book(name) - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this = cust_mod._cust_rev(yf, yt, team_id, book) - last = cust_mod._cust_rev(lf, lt, team_id, book) - lastord = cust_mod._last_order_dates(None, None, team_id, book) # all-time last order = recency - ids = list(book) if book is not None else list(set(this) | set(last)) - attrs = cust_mod._partner_attrs(set(ids)) - namemap = {r['id']: r.get('name') for r in O.search_read('res.partner', [('id', 'in', ids)], ['name'])} - rows = [] - for p in ids: - tr = this.get(p, {}).get('rev', 0.0) - lr = last.get(p, {}).get('rev', 0.0) - a = attrs.get(p, {}) - lo = lastord.get(p, '') - recency = (t - dt.date.fromisoformat(lo)).days if lo else None - status = 'Active' if tr > 0 else ('Dormant' if (lr > 0 or lo) else 'Inactive') - rows.append({'pid': p, 'customer': namemap.get(p) or (this.get(p) or last.get(p) or {}).get('name', '?'), - 'rev_ytd': tr, 'rev_ly': lr, 'change': tr - lr, 'yoy_pct': P.yoy_pct(tr, lr), - 'orders': this.get(p, {}).get('orders', 0), 'last_order': lo, 'recency_days': recency, - 'status': status, 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'), - 'agent': a.get('agent', name)}) - rows.sort(key=lambda r: (r['rev_ytd'] <= 0, -r['rev_ytd'], -(r['rev_ly']))) - return rows - - -# ------------------------------------------------------------ top SKUs (with profit/order) -def top_skus(name, t=None, team_id=None, top=30): - """The book's top SKUs YTD (line-level), each with revenue, units, margin and profit/order.""" - t = t or P.today() - book = _book(name) - if book is not None and not book: - return [] - yf, yt = P.ytd(t) - lex = [('order_partner_id', 'in', list(book))] if book is not None else None - return sales_mod.decompose(yf, yt, team_id, line_extra=lex, top=top)['skus'] - - -# ------------------------------------------------------------ returns (book-scoped, consolidated) -def returns_trend(name, n=13, t=None): - return sales_mod.returns_monthly(n, t, partner_ids=_book(name)) - - -def returns_headline(name, t=None): - return sales_mod.returns_headline(t, partner_ids=_book(name)) - - -# ------------------------------------------------------------ all-agents rollup (the page table) -def _returns_by_agent(t=None): - """{agent_name: returns$ YTD} — credit notes mapped to each customer's agent.""" - by_p = sales_mod.returns_by_partner(t) - attrs = cust_mod._partner_attrs(list(by_p)) - agg = {} - for pid, amt in by_p.items(): - a = (attrs.get(pid) or {}).get('agent') or '(none)' - agg[a] = agg.get(a, 0.0) + amt - return agg - - -def rollup(t=None, team_id=None): - """Every agent ranked by YTD book revenue (+ YoY, customers, orders) with returns $ and return - rate. Reuses the Customers MECE agent rollup, so Σ(agents) == total YTD revenue.""" - rows = cust_mod.by_dimension('agent', t, team_id=team_id) - ret = _returns_by_agent(t) - for r in rows: - r['agent'] = r['group'] - r['returns'] = ret.get(r['group'], 0.0) - r['return_rate_pct'] = (r['returns'] / r['revenue'] * 100.0) if r.get('revenue') else 0.0 - return rows - - -# ------------------------------------------------------------ VALIDATION -def validate(t=None, team_id=None): - """Reconcile the agent rollup to Odoo. (1) Σ(agent book revenue) == total YTD revenue — the - rollup is MECE over customers. (2) A sampled agent's scorecard YTD == its headline YTD.""" - t = t or P.today() - yf, yt = P.ytd(t) - checks = [] - rows = rollup(t, team_id=team_id) - agent_sum = sum(r['revenue'] for r in rows) - total = O.sum_field('sale.order', sales_mod.order_domain(yf, yt, team_id), 'amount_untaxed') - checks.append({'check': 'YTD revenue: Σ(agent book) == total', 'a': round(agent_sum, 2), - 'b': round(total, 2), 'gap': round(agent_sum - total, 2), - 'ok': abs(agent_sum - total) <= max(1.0, 0.001 * (total or 1))}) - # sampled agent: scorecard YTD == headline YTD for the same window - sample = next((r['agent'] for r in rows if r['agent'] not in ('(none)',)), None) - if sample: - sc_ytd = next((s['revenue'] for s in scorecard(sample, t, team_id) if s['key'] == 'ytd'), 0.0) - hl = headline(sample, yf, yt, team_id)['revenue'] - checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2), - 'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0}) - return checks - - -# ------------------------------------------------------------ INVOICE-LINE ATTRIBUTION (2026-07-28) -# A SECOND agent source. Everything above this line attributes by BOOK — the customer's assigned -# agent (res.partner.agent_ids) — over confirmed SALES ORDERS. This section attributes per INVOICE -# LINE, from the OCA sale-commission module, via the semantic layer (topics invoice_lines / -# commission_lines). The two disagree on purpose and answer different questions: -# -# book -> "whose customer is this / who owns the relationship" (order basis) -# invoice -> "what was actually credited to an agent on the billing" + the ONLY source that can -# say what is NOT allocated to an agent (invoice basis) -# -# ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT — `res.partner.agent` is the flag. -# "Anna" and "Shantal Erlich" are internal SALESPEOPLE who carry commission lines; the `agent` -# dim excludes them and `include_salespeople` folds them back in as a clearly-labelled variant. -# See [[invoice-line-agent-commission]]. - -_ALLOC_LABEL = {'agent': 'Allocated to an agent', 'salesperson': 'Salesperson only', - 'none': 'Not allocated'} - - -def invoice_line_rollup(t=None, team_id=None, include_salespeople=False): - """Per-name invoice-line revenue + the MECE allocation split, for a YTD window. - - Returns {'by_agent': [...], 'allocation': [...], 'total': float, 'allocated': float, - 'unallocated': float, 'basis': str} — or {'error': msg} when the tenant store is not - ready (this path is store-only; there is no live fallback that stays honest about the - unallocated bucket). - """ - import harness.semantic as S - t = t or P.today() - yf, yt = P.ytd(t) - dim = 'commission_name' if include_salespeople else 'agent' - try: - by = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=[dim], - date_from=yf, date_to=yt, team_id=team_id, limit=200).get('rows') or [] - alloc = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=['allocation'], - date_from=yf, date_to=yt, team_id=team_id, limit=10).get('rows') or [] - tot = (S.store_query('invoice_lines', ['invoiced_line_sales'], date_from=yf, date_to=yt, - team_id=team_id).get('rows') or [{}])[0].get('invoiced_line_sales') or 0.0 - except Exception as e: # store not ready / model error — say so, don't fake - return {'error': str(e)} - # store_query row shape: the DIM KEY carries the display NAME and `_id` the raw value - # (allocation -> 'Salesperson only', allocation_id -> 'salesperson'). Reading `_name` - # returns None for every row and silently renders the whole table as "no agent". - rows = [{'agent': (r.get(dim) or '(no agent on the line)'), - 'revenue': r.get('invoiced_line_sales') or 0.0} for r in by] - rows.sort(key=lambda r: -r['revenue']) - amap = {r.get('allocation_id') or 'none': (r.get('invoiced_line_sales') or 0.0) for r in alloc} - allocated = amap.get('agent', 0.0) - return { - 'by_agent': rows, - 'allocation': [{'bucket': _ALLOC_LABEL[k], 'revenue': amap.get(k, 0.0)} - for k in ('agent', 'salesperson', 'none') if k in amap or True], - 'total': tot, 'allocated': allocated, 'unallocated': tot - allocated, - 'strict_none': amap.get('none', 0.0), 'salesperson_only': amap.get('salesperson', 0.0), - 'window': (yf, yt), - 'basis': ('invoice line · commission names incl. salespeople' if include_salespeople - else 'invoice line · real agents only'), - } +"""Agent module — per-agent (res.partner.agent_ids) analytics. + +An *agent* owns a **book** of customers (the same attribute the Customers module slices by). This +module reports that book the way Sales/Customers/SKU report the whole company: a period scorecard +with custom date windows (Today / WTD / Last week / MTD / QTD / YTD / any custom range), a sales +trend, returns, top SKUs (with profit/order) and the FULL customer list — INCLUDING inactive +accounts (no recent orders) so a rep sees who they've stopped selling to. + +Scope: reuses the Sales `order_domain` (Fisch+Royal, excluded accounts removed, state sale/done) +so numbers tie to every other module. Returns are consolidated (credit notes aren't BU-tagged); +everything else is BU-filterable via team_id. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import core.odoo as O +import core.periods as P +import modules.sales as sales_mod +import modules.customers as cust_mod + + +def options(t=None, team_id=None): + """Agent names with book activity — for the page/drawer picker.""" + return cust_mod.agent_options(t, team_id) + + +def _book(name): + """frozenset of every partner id assigned to the agent (incl. inactive). None only for 'All'.""" + return cust_mod.agent_partner_ids(name) + + +def _rev(date_from, date_to, team_id, book): + return O.sum_field('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), + 'amount_untaxed') + + +def _orders(date_from, date_to, team_id, book): + return O.get_odoo().search_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book)) + + +def _custs(date_from, date_to, team_id, book): + return O.distinct_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), + 'partner_id') + + +# ------------------------------------------------------------ period scorecard (custom dating) +# Today / WTD / Last week / MTD / QTD / YTD — each carries its window so the UI can decompose it +# and so "what did this agent sell this/last week" is a click, not a date-math exercise. +_PERIODS = [('Today', 'today'), ('Week to date', 'wtd'), ('Last week', 'lwk'), + ('Month to date', 'mtd'), ('Quarter to date', 'qtd'), ('Year to date', 'ytd')] + + +def _window(key, t): + if key == 'today': + return P._d(t), P._d(t) + if key == 'lwk': # the full prior Mon–Sun week + f, _tt = P.wtd(t) + start = dt.date.fromisoformat(f) - dt.timedelta(days=7) + return start.isoformat(), (dt.date.fromisoformat(f) - dt.timedelta(days=1)).isoformat() + return {'wtd': P.wtd, 'mtd': P.mtd, 'qtd': P.qtd, 'ytd': P.ytd}[key](t) + + +def scorecard(name, t=None, team_id=None): + """Book revenue (+ YoY same-period), orders for Today / WTD / Last week / MTD / QTD / YTD.""" + t = t or P.today() + book = _book(name) + out = [] + for label, key in _PERIODS: + f, tt = _window(key, t) + wk = key in ('today', 'wtd', 'lwk') # weekday-align the short windows' LY compare + cf, ct = P.shift_year(f, tt, weeks=wk) + rev, rev_ly = _rev(f, tt, team_id, book), _rev(cf, ct, team_id, book) + out.append({'key': key, 'label': label, 'date_from': f, 'date_to': tt, 'cmp_from': cf, 'cmp_to': ct, + 'revenue': rev, 'revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly), + 'orders': _orders(f, tt, team_id, book)}) + return out + + +def headline(name, date_from, date_to, team_id=None): + """Book KPIs for an ARBITRARY window (custom dating): revenue + YoY (same window LY), orders, + active customers, AOV and returns $ / return rate.""" + book = _book(name) + cf, ct = P.shift_year(date_from, date_to, weeks=False) + rev, rev_ly = _rev(date_from, date_to, team_id, book), _rev(cf, ct, team_id, book) + orders = _orders(date_from, date_to, team_id, book) + ret = sales_mod._returns_amt(date_from, date_to, book) + return {'date_from': date_from, 'date_to': date_to, 'cmp_from': cf, 'cmp_to': ct, + 'revenue': rev, 'revenue_ly': rev_ly, + 'yoy_pct': P.yoy_pct(rev, rev_ly), 'orders': orders, + 'customers': _custs(date_from, date_to, team_id, book), 'aov': (rev / orders) if orders else 0.0, + 'returns': ret, 'return_rate_pct': (ret / rev * 100.0) if rev else 0.0} + + +# ------------------------------------------------------------ sales trend +def _book_monthly_rev(book, date_from, date_to, team_id=None): + g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book), + ['amount_untaxed:sum'], ['date_order:month'], lazy=False) + out = {} + for r in g: + ym = ((r.get('__range') or {}).get('date_order:month') or {}).get('from', '')[:7] + if ym: + out[ym] = r.get('amount_untaxed') or 0.0 + return out + + +def monthly(name, n=13, t=None, team_id=None): + """Book sales per month vs the same month last year (one month-grouped query over 2 years).""" + t = t or P.today() + book = _book(name) + mrev = _book_monthly_rev(book, dt.date(t.year - 2, t.month, 1).isoformat(), t.isoformat(), team_id) + rows = [] + for ym, _s, _e in P.month_starts(n, t): + y, m = int(ym[:4]) - 1, int(ym[5:7]) + this, last = mrev.get(ym, 0.0), mrev.get(f'{y:04d}-{m:02d}', 0.0) + rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)}) + return rows + + +# ------------------------------------------------------------ full customer book (incl. inactive) +def customers(name, t=None, team_id=None): + """EVERY customer in the agent's book, including inactive accounts (no YTD/LY orders) — those + show $0 with status 'Inactive'/'Dormant'. Each row is clickable to the customer drawer and + carries recency so a rep can see who's gone quiet. Sorted by YTD revenue desc (inactive last).""" + t = t or P.today() + book = _book(name) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = cust_mod._cust_rev(yf, yt, team_id, book) + last = cust_mod._cust_rev(lf, lt, team_id, book) + lastord = cust_mod._last_order_dates(None, None, team_id, book) # all-time last order = recency + ids = list(book) if book is not None else list(set(this) | set(last)) + attrs = cust_mod._partner_attrs(set(ids)) + namemap = {r['id']: r.get('name') for r in O.search_read('res.partner', [('id', 'in', ids)], ['name'])} + rows = [] + for p in ids: + tr = this.get(p, {}).get('rev', 0.0) + lr = last.get(p, {}).get('rev', 0.0) + a = attrs.get(p, {}) + lo = lastord.get(p, '') + recency = (t - dt.date.fromisoformat(lo)).days if lo else None + status = 'Active' if tr > 0 else ('Dormant' if (lr > 0 or lo) else 'Inactive') + rows.append({'pid': p, 'customer': namemap.get(p) or (this.get(p) or last.get(p) or {}).get('name', '?'), + 'rev_ytd': tr, 'rev_ly': lr, 'change': tr - lr, 'yoy_pct': P.yoy_pct(tr, lr), + 'orders': this.get(p, {}).get('orders', 0), 'last_order': lo, 'recency_days': recency, + 'status': status, 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'), + 'agent': a.get('agent', name)}) + rows.sort(key=lambda r: (r['rev_ytd'] <= 0, -r['rev_ytd'], -(r['rev_ly']))) + return rows + + +# ------------------------------------------------------------ top SKUs (with profit/order) +def top_skus(name, t=None, team_id=None, top=30): + """The book's top SKUs YTD (line-level), each with revenue, units, margin and profit/order.""" + t = t or P.today() + book = _book(name) + if book is not None and not book: + return [] + yf, yt = P.ytd(t) + lex = [('order_partner_id', 'in', list(book))] if book is not None else None + return sales_mod.decompose(yf, yt, team_id, line_extra=lex, top=top)['skus'] + + +# ------------------------------------------------------------ returns (book-scoped, consolidated) +def returns_trend(name, n=13, t=None): + return sales_mod.returns_monthly(n, t, partner_ids=_book(name)) + + +def returns_headline(name, t=None): + return sales_mod.returns_headline(t, partner_ids=_book(name)) + + +# ------------------------------------------------------------ all-agents rollup (the page table) +def _returns_by_agent(t=None): + """{agent_name: returns$ YTD} — credit notes mapped to each customer's agent.""" + by_p = sales_mod.returns_by_partner(t) + attrs = cust_mod._partner_attrs(list(by_p)) + agg = {} + for pid, amt in by_p.items(): + a = (attrs.get(pid) or {}).get('agent') or '(none)' + agg[a] = agg.get(a, 0.0) + amt + return agg + + +def rollup(t=None, team_id=None): + """Every agent ranked by YTD book revenue (+ YoY, customers, orders) with returns $ and return + rate. Reuses the Customers MECE agent rollup, so Σ(agents) == total YTD revenue.""" + rows = cust_mod.by_dimension('agent', t, team_id=team_id) + ret = _returns_by_agent(t) + for r in rows: + r['agent'] = r['group'] + r['returns'] = ret.get(r['group'], 0.0) + r['return_rate_pct'] = (r['returns'] / r['revenue'] * 100.0) if r.get('revenue') else 0.0 + return rows + + +# ------------------------------------------------------------ VALIDATION +def validate(t=None, team_id=None): + """Reconcile the agent rollup to Odoo. (1) Σ(agent book revenue) == total YTD revenue — the + rollup is MECE over customers. (2) A sampled agent's scorecard YTD == its headline YTD.""" + t = t or P.today() + yf, yt = P.ytd(t) + checks = [] + rows = rollup(t, team_id=team_id) + agent_sum = sum(r['revenue'] for r in rows) + total = O.sum_field('sale.order', sales_mod.order_domain(yf, yt, team_id), 'amount_untaxed') + checks.append({'check': 'YTD revenue: Σ(agent book) == total', 'a': round(agent_sum, 2), + 'b': round(total, 2), 'gap': round(agent_sum - total, 2), + 'ok': abs(agent_sum - total) <= max(1.0, 0.001 * (total or 1))}) + # sampled agent: scorecard YTD == headline YTD for the same window + sample = next((r['agent'] for r in rows if r['agent'] not in ('(none)',)), None) + if sample: + sc_ytd = next((s['revenue'] for s in scorecard(sample, t, team_id) if s['key'] == 'ytd'), 0.0) + hl = headline(sample, yf, yt, team_id)['revenue'] + checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2), + 'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0}) + checks.extend(validate_measures(t=t, team_id=team_id)) + return checks + + +def validate_measures(t=None, team_id=None, days=90): + """⭐⭐ W37-T12 — the minted PER-AGENT lookback columns, against a DIRECT Odoo aggregate. + + ⛔ WHICH AGENT SOURCE, AND THE TICKET REQUIRES IT SAID OUT LOUD: this reconciles ROUTE 1, the + CUSTOMER-MASTER BOOK (`res.partner.agent_ids` -> the mirror's `res_partner.agent_id`, which is + `sales_lines`' `agent` dim). It is NOT the OCA commission route below — measured 2026-08-19, + 9 agents carry route-1 revenue against 12 on commission lines and 17 carrying the flag, so the + two produce materially different rankings and a check that mixed them would be comparing two + different questions and calling the gap an error. + + ⚠ THE ORACLE IS THE ORDER HEADER, not the mirror the columns are served from. `sale.order` + grouped by `partner_id`, mapped to each customer's agent CLIENT-SIDE — a different model, a + different grain and a different code path from `store_query`'s line-level sum, so agreement + between them is evidence rather than tautology. + ⚠ Windowed to the MIRROR'S newest order, like `product_data.validate_measures`, so the + residual is about EDITS to a shared period and not about orders the mirror has never seen. + """ + from harness import datastore as DS + from harness import semantic as sem + + t = t or P.today() + checks = [] + try: + if not DS.ready(): + return [{'check': 'agent lookback measures reconcile to Odoo', 'a': 'no mirror', + 'b': '-', 'ok': False, + 'detail': 'the tenant store is not readable, so this is UNPROVEN, which standing rule ' + '8 does not accept as green'}] + except Exception as e: # noqa: BLE001 + return [{'check': 'agent lookback measures reconcile to Odoo', 'a': type(e).__name__, + 'b': '-', 'ok': False, 'detail': str(e)[:200]}] + + offer = sem.entity_measures('odoo_agents') + checks.append({'check': 'the agent measure OFFER is non-empty and every key resolves ' + '(owner item 4 / R1)', + 'a': len(offer), 'b': '>0', 'ok': bool(offer), + 'detail': {'keys': [m['key'] for m in offer], + 'refused': sem.entity_measure_refusals('odoo_agents')}}) + if not offer: + return checks + + con = DS.ro_cursor() + try: + newest = con.execute('SELECT max(date_order) FROM sale_order').fetchone() + finally: + con.close() + d_to = t - dt.timedelta(days=2) + if newest and newest[0]: + try: + d_to = min(d_to, dt.date.fromisoformat(str(newest[0])[:10]) - dt.timedelta(days=1)) + except ValueError: + pass + d_from = d_to - dt.timedelta(days=days) + DF, DT = d_from.isoformat(), d_to.isoformat() + + ours = sem.entity_measure_values('odoo_agents', ['revenue'], date_from=DF, date_to=DT, + team_id=team_id, offer=offer) + # THE ORACLE — order headers, grouped by customer, mapped to that customer's agent here. + o = O.get_odoo() + grp = o.read_group('sale.order', sales_mod.order_domain(DF, DT, team_id), + ['partner_id', 'amount_untaxed:sum'], ['partner_id'], lazy=False) + pids = sorted({r['partner_id'][0] for r in grp if r.get('partner_id')}) + agent_of = {} + for i in range(0, len(pids), 500): + for p in o.search_read('res.partner', [('id', 'in', pids[i:i + 500])], + ['id', 'agent_ids']): + ag = (p.get('agent_ids') or []) + if ag: + agent_of[p['id']] = ag[0] # the Customers-module convention: agent_ids[0] + theirs = {} + for r in grp: + if not r.get('partner_id'): + continue + a = agent_of.get(r['partner_id'][0]) + if a is not None: + theirs[a] = theirs.get(a, 0.0) + r['amount_untaxed'] + + ours_tot = round(sum(c.get('revenue', 0) for c in ours.values()), 2) + theirs_tot = round(sum(theirs.values()), 2) + # ⚠ ORDER-HEADER vs LINE-SUM is a REAL basis difference (an order's untaxed total includes + # lines this topic's service filter drops), so the tolerance is a stated 3% rather than a + # cent — and the FIGURE is reported so a drift is readable instead of absorbed. + gap = ours_tot - theirs_tot + checks.append({ + 'check': 'per-agent revenue (BOOK route) vs an INDEPENDENT Odoo order-header aggregate ' + 'mapped through res.partner.agent_ids', + 'a': ours_tot, 'b': theirs_tot, 'gap': round(gap, 2), + 'ok': bool(theirs_tot) and abs(gap) <= 0.03 * theirs_tot, + 'detail': {'window': [DF, DT], 'agents_ours': len(ours), 'agents_theirs': len(theirs), + 'gap_pct': round(gap / theirs_tot * 100, 3) if theirs_tot else None, + 'route': 'customer-master book (res.partner.agent_ids), NOT the OCA ' + 'commission table; the two name different people'}}) + # ⛔ AND PER AGENT, because a total can agree while every row is keyed wrong — the join-key + # trap contract C1 names. Here the key is the `res.partner` id at both ends. + off_by = sorted(((abs(ours.get(a, {}).get('revenue', 0.0) - v), a) + for a, v in theirs.items() if v), reverse=True) + bad = [(a, round(ours.get(a, {}).get('revenue', 0.0), 2), round(theirs[a], 2)) + for d, a in off_by if d > 0.03 * theirs[a]] + checks.append({ + 'check': 'each agent\'s own figure ties (the C1 join-key test: a wrong key agrees in ' + 'total and disagrees on every row)', + 'a': len(bad), 'b': 0, 'ok': not bad, + 'detail': {'worst': [(a, ours_v, th_v) for a, ours_v, th_v in bad[:5]], + 'agents_compared': len(theirs)}}) + return checks + + +# ------------------------------------------------------------ INVOICE-LINE ATTRIBUTION (2026-07-28) +# A SECOND agent source. Everything above this line attributes by BOOK — the customer's assigned +# agent (res.partner.agent_ids) — over confirmed SALES ORDERS. This section attributes per INVOICE +# LINE, from the OCA sale-commission module, via the semantic layer (topics invoice_lines / +# commission_lines). The two disagree on purpose and answer different questions: +# +# book -> "whose customer is this / who owns the relationship" (order basis) +# invoice -> "what was actually credited to an agent on the billing" + the ONLY source that can +# say what is NOT allocated to an agent (invoice basis) +# +# ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT — `res.partner.agent` is the flag. +# "Anna" and "Shantal Erlich" are internal SALESPEOPLE who carry commission lines; the `agent` +# dim excludes them and `include_salespeople` folds them back in as a clearly-labelled variant. +# See [[invoice-line-agent-commission]]. + +_ALLOC_LABEL = {'agent': 'Allocated to an agent', 'salesperson': 'Salesperson only', + 'none': 'Not allocated'} + + +def invoice_line_rollup(t=None, team_id=None, include_salespeople=False): + """Per-name invoice-line revenue + the MECE allocation split, for a YTD window. + + Returns {'by_agent': [...], 'allocation': [...], 'total': float, 'allocated': float, + 'unallocated': float, 'basis': str} — or {'error': msg} when the tenant store is not + ready (this path is store-only; there is no live fallback that stays honest about the + unallocated bucket). + """ + import harness.semantic as S + t = t or P.today() + yf, yt = P.ytd(t) + dim = 'commission_name' if include_salespeople else 'agent' + try: + by = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=[dim], + date_from=yf, date_to=yt, team_id=team_id, limit=200).get('rows') or [] + alloc = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=['allocation'], + date_from=yf, date_to=yt, team_id=team_id, limit=10).get('rows') or [] + tot = (S.store_query('invoice_lines', ['invoiced_line_sales'], date_from=yf, date_to=yt, + team_id=team_id).get('rows') or [{}])[0].get('invoiced_line_sales') or 0.0 + except Exception as e: # store not ready / model error — say so, don't fake + return {'error': str(e)} + # store_query row shape: the DIM KEY carries the display NAME and `_id` the raw value + # (allocation -> 'Salesperson only', allocation_id -> 'salesperson'). Reading `_name` + # returns None for every row and silently renders the whole table as "no agent". + rows = [{'agent': (r.get(dim) or '(no agent on the line)'), + 'revenue': r.get('invoiced_line_sales') or 0.0} for r in by] + rows.sort(key=lambda r: -r['revenue']) + amap = {r.get('allocation_id') or 'none': (r.get('invoiced_line_sales') or 0.0) for r in alloc} + allocated = amap.get('agent', 0.0) + return { + 'by_agent': rows, + 'allocation': [{'bucket': _ALLOC_LABEL[k], 'revenue': amap.get(k, 0.0)} + for k in ('agent', 'salesperson', 'none') if k in amap or True], + 'total': tot, 'allocated': allocated, 'unallocated': tot - allocated, + 'strict_none': amap.get('none', 0.0), 'salesperson_only': amap.get('salesperson', 0.0), + 'window': (yf, yt), + 'basis': ('invoice line · commission names incl. salespeople' if include_salespeople + else 'invoice line · real agents only'), + } diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 0ff439a38cb7a3dc51951e01cfc34e14e911815a..e32e7fe797b0592beb6702533f9b2e51025b1f28 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -51,6 +51,7 @@ template and this deliberately mirrors its signature and its row shape. carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two sources are JOINED rather than merged. """ +import datetime as _dt import json import zlib from pathlib import Path @@ -391,6 +392,17 @@ def pool(team_id=None, t=None): bu_share = {} if consolidated else _bu_ltm_share(t, team_id) sup = supplier_master() prices, _price_report = _pricelist_by_code() + # ⭐ W37-T14 / T15. ⚠ MEASURED COST, stated because it lands on a scope's FIRST build: + # tier prices 15.1 s + packagings 8.1 s on top of the ~44 s consolidated build. Only the + # first build for a scope blocks (`routes_products._pool_for`'s stale-while-refresh), and + # both degrade to `{}` on a read failure rather than taking the grid down — the same + # asymmetry `pricelist_by_code` documents: a column is not the ROW SET. + try: + _prods = products.active_products() # ONE read, shared by both (see its docstring) + except Exception: # noqa: BLE001 + _prods = None + tiers, _tier_report = products.tier_prices_by_code(_prods) + packs, _pack_report = products.packagings_by_code(_prods) cat = _catalogue_by_code() # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those @@ -437,6 +449,19 @@ def pool(team_id=None, t=None): p = prices.get(code) or {} for _col, _name in products.PRICELIST_COLUMNS: row[_col] = p.get(_col) + # ⭐⭐ W37-T14 / T15 — THE HONEST SETS, beside the three declared columns rather than + # instead of them. The columns above answer "what does Fisch charge"; these answer "how + # many prices/units does this SKU actually have", which the columns structurally cannot: + # they are named after THIS tenant's lists, so a price on any other list is invisible. + # ⚠ A `json` cell is a STRING on the wire (the type's own contract), so these are dumped + # here rather than handed over as lists — a bare list would round-trip through the overlay + # as something `grid_events` refuses to re-parse. + # ⚠ BLANK, NOT "[]" — an empty cell reads as "sold in one unit / not priced on any list", + # and an empty JSON array on screen reads as a bug. + _tiers = tiers.get(code) + row["tier_prices"] = json.dumps(_tiers, ensure_ascii=False) if _tiers else None + _units = packs.get(code) + row["units"] = json.dumps(_units, ensure_ascii=False) if _units else None # Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock). s = sup.get(code) or {} row.update({ @@ -801,4 +826,390 @@ def validate(team_id=None, t=None): # have silently dropped. 0 today does not make that filter correct. "cover_gap_rounds_to_zero": len(rounding_would_miss)}, }) + checks.extend(validate_measures(t=t, team_id=team_id, + pool_codes={r["code"] for r in rows})) + checks.extend(validate_price_and_unit_cells(rows)) + checks.extend(validate_stock_measures(t=t)) + return checks + + +def validate_stock_measures(t=None, days=90, sample=4): + """⭐⭐ W37-T13 — `stock_in` / `stock_out` per SKU, against a DIRECT live Odoo `read_group`. + + ⛔ THE TWO-SIDED DOMAIN IS REPRODUCED ON THE LIVE SIDE, and getting it wrong there would hide + exactly the defect this validates. `location_id.usage` is a DOT-PATH FILTER, which Odoo + supports; a dot-path GROUPBY faults, which is why the direction is expressed as two separate + filtered reads rather than one grouped-by-usage read (`proto/P1-stock-moves.md`, gotcha 1). + + ⛔ AND THE DIRECTION SPLIT IS ASSERTED, NOT ASSUMED. A one-sided domain produces IN == OUT for + every internal transfer, so a run where the two columns agree everywhere is the signature of + the bug rather than of a quiet warehouse. The last leg requires a SKU where they genuinely + differ — the ticket's own `done-when` clause, and the only one a total cannot fake. + """ + from harness import datastore as DS + from harness import semantic as sem + + t = t or P.today() + checks = [] + offer = sem.entity_measures("odoo_products") + keys = [m["key"] for m in offer if m["key"].startswith("stock_")] + if not keys: + why = [r for r in sem.entity_measure_refusals("odoo_products") + if str(r.get("key", "")).startswith("stock_")] + return [{"check": "the product catalogue offers Stock moved in / out (W37-T13)", + "ours": 0, "theirs": 3, "ok": False, + # ⭐ The refusal carries its own cause — reported, not inferred from an absence. + "detail": {"refusals": why or "no stock binding declared"}}] + checks.append({"check": "the product measure catalogue offers the stock-movement keys " + "(W37-T13)", "ours": sorted(keys), "theirs": 3, + "ok": {"stock_in", "stock_out"} <= set(keys)}) + + con = DS.ro_cursor() + try: + newest = con.execute("SELECT max(date) FROM stock_move").fetchone() + finally: + con.close() + d_to = t - _dt.timedelta(days=2) + if newest and newest[0]: + try: + d_to = min(d_to, _dt.date.fromisoformat(str(newest[0])[:10]) - _dt.timedelta(days=1)) + except ValueError: + pass + d_from = d_to - _dt.timedelta(days=days) + DF, DT = d_from.isoformat(), d_to.isoformat() + ours = sem.entity_measure_values("odoo_products", ["stock_in", "stock_out"], + date_from=DF, date_to=DT, offer=offer) + + o = O.get_odoo() + base = [("state", "=", "done"), + ("date", ">=", f"{DF} 00:00:00"), ("date", "<=", f"{DT} 23:59:59")] + IN = base + [("location_dest_id.usage", "=", "internal"), + ("location_id.usage", "!=", "internal")] + OUT = base + [("location_id.usage", "=", "internal"), + ("location_dest_id.usage", "!=", "internal")] + live = {} + for dom, side in ((IN, "stock_in"), (OUT, "stock_out")): + for r in o.read_group("stock.move", dom, ["product_id", "quantity_done:sum"], + ["product_id"], lazy=False): + if not r.get("product_id"): + continue + live.setdefault(r["product_id"][0], {})[side] = r["quantity_done"] + code_of = _codes_of_odoo_products(o, list(live)) + by_code = {} + for pid_, v in live.items(): + c = code_of.get(pid_, f"pid:{pid_}") + d = by_code.setdefault(c, {"stock_in": 0.0, "stock_out": 0.0}) + for k in ("stock_in", "stock_out"): + d[k] += v.get(k, 0.0) + + for side in ("stock_in", "stock_out"): + a = round(sum(c.get(side, 0) for c in ours.values()), 2) + b = round(sum(c.get(side, 0) for c in by_code.values()), 2) + checks.append({ + "check": f"{side}: the mirror's per-SKU total vs a DIRECT Odoo read_group under the " + f"SAME two-sided location domain", + "ours": a, "theirs": b, "gap": round(a - b, 2), + "ok": bool(b) and abs(a - b) <= 0.02 * b, + "detail": {"window": [DF, DT], "skus_ours": len(ours), "skus_odoo": len(by_code)}}) + + # ⛔ THE NAMED SKU, and it is chosen for DIFFERING — see the docstring. + diff = sorted(((abs((c.get("stock_in") or 0) - (c.get("stock_out") or 0)), k) + for k, c in ours.items()), reverse=True)[:sample] + named = [] + for _d, k in diff: + named.append({"sku": k, + "ours": {s: round(ours[k].get(s, 0), 2) for s in ("stock_in", "stock_out")}, + "odoo": {s: round((by_code.get(k) or {}).get(s, 0), 2) + for s in ("stock_in", "stock_out")}}) + off = [n for n in named + if any(abs(n["ours"][s] - n["odoo"][s]) > max(0.01, 0.02 * (n["odoo"][s] or 1)) + for s in ("stock_in", "stock_out"))] + checks.append({ + "check": f"each NAMED SKU's in/out ties to Odoo ({len(named)} SKUs, picked for the " + f"largest in-vs-out difference)", + "ours": len(off), "theirs": 0, "ok": not off, + "detail": {"named": named[:3], "mismatched": off[:2]}}) + genuinely_split = [n for n in named if n["ours"]["stock_in"] != n["ours"]["stock_out"]] + checks.append({ + "check": "⛔ the DIRECTION SPLIT is real: at least one SKU where IN and OUT genuinely " + "differ. A one-sided domain makes them equal for every internal transfer, so " + "all-equal is the SIGNATURE OF THE BUG, not a quiet warehouse", + "ours": len(genuinely_split), "theirs": ">=1", "ok": bool(genuinely_split), + "detail": {"example": genuinely_split[0] if genuinely_split else None}}) + return checks + + +def validate_price_and_unit_cells(rows, sample=6): + """⭐⭐ W37-T14 / T15 — the `Tier prices` and `Units` cells, against a FRESH Odoo read. + + ⛔ THE ORACLE IS ASKED PER SKU, not in bulk, and deliberately so: the builders group a + bulk read in Python, so re-running the same bulk read would re-run the same grouping and + could only ever agree with itself. Asking Odoo for ONE SKU's price rules is a different + question shape and can actually disagree ([[no-unverifiable-aggregates]]). + + ⚠ WHAT IS NOT PROVEN HERE, said rather than implied: that a PERSON sees the cell. These are + `json` columns on the product grid and the render is the client's; the data half is what a + module `validate()` can reach. + """ + checks = [] + priced = [r for r in rows if r.get("tier_prices")] + united = [r for r in rows if r.get("units")] + checks.append({ + "check": "the product grid serves a Tier-prices cell (W37-T14) and a Units cell (T15); " + "a DECLARED column that is never filled is the defect these replace", + "ours": {"with_tier_prices": len(priced), "with_units": len(united), "rows": len(rows)}, + "theirs": ">0 each", + # ⚠ Units are legitimately sparse (19.6% measured), so the floor is existence, not a rate. + "ok": bool(priced) and bool(united), + "detail": {"multi_price_skus": sum(1 for r in priced + if len(json.loads(r["tier_prices"])) > 1)}, + }) + if not priced: + return checks + o = O.get_odoo() + pls = {p["id"]: str(p.get("name") or "").strip() + for p in O.search_read("product.pricelist", [], ["id", "name"])} + today = P.today().isoformat() + # Prefer SKUs that carry MORE THAN ONE price — the ticket's own subject. + cand = sorted(priced, key=lambda r: -len(json.loads(r["tier_prices"])))[:sample] + bad = [] + for r in cand: + mine = sorted((t["pricelist"], round(float(t["unit_price"]), 2)) + for t in json.loads(r["tier_prices"])) + pid = r.get("product_id") + tmpl = None + if pid: + rec = o.search_read("product.product", [("id", "=", pid)], ["product_tmpl_id"]) + tmpl = O.m2o_id(rec[0].get("product_tmpl_id")) if rec else None + dom = [("compute_price", "=", "fixed"), + "|", ("date_start", "=", False), ("date_start", "<=", today), + "|", ("date_end", "=", False), ("date_end", ">=", today), + ("fixed_price", ">", 0), + "|", ("product_id", "=", pid), ("product_tmpl_id", "=", tmpl)] + live = o.search_read("product.pricelist.item", dom, + ["pricelist_id", "fixed_price", "min_quantity", "applied_on"]) + best = {} + for it in live: + nm = pls.get(O.m2o_id(it.get("pricelist_id")), "?") + q = it.get("min_quantity") or 0.0 + if nm not in best or q < best[nm][0]: + best[nm] = (q, round(it.get("fixed_price") or 0.0, 2)) + theirs = sorted((nm, v) for nm, (q, v) in best.items()) + # ⚠ A code carried by TWO active products legitimately holds MORE entries than a single + # product's rules (D-309 / `2112-12`), so ours is a SUPERSET, never an equality. + if not set(theirs) <= set(mine): + bad.append({"sku": r.get("code"), "ours": mine, "odoo": theirs}) + checks.append({ + "check": f"each sampled SKU's Tier prices contain every live Odoo price for it " + f"({len(cand)} SKUs, chosen for having the MOST prices)", + "ours": len(bad), "theirs": 0, "ok": not bad, + "detail": {"mismatches": bad[:3], + "sampled": [r.get("code") for r in cand]}, + }) + return checks + + +def _codes_of_odoo_products(o, ids): + """`{odoo product id: the identity key the GRID uses}` — `default_code`, or `pid:`. + + ⚠ BATCHED, 500 at a time. One call per id is ~1,700 XML-RPC round trips on this window and + turns a 5-second reconciliation into a coffee break. + ⚠ `active in [True, False]`: a re-SKUed line points at the ARCHIVED record and the grid + merges it under the surviving code, so an active-only read would key it `pid:` and + manufacture a mismatch this check would then report as a defect. + """ + ids = sorted({i for i in ids if i}) + out = {} + for i in range(0, len(ids), 500): + for p in o.search_read('product.product', + [('id', 'in', ids[i:i + 500]), ('active', 'in', [True, False])], + ['default_code']): + out[p['id']] = p.get('default_code') or f"pid:{p['id']}" + return {i: out.get(i, f"pid:{i}") for i in ids} + + +def validate_measures(t=None, team_id=None, days=90, pool_codes=None): + """⭐⭐ W37-T10 — THE MINTED LOOKBACK MEASURES, against a DIRECT Odoo `read_group`. + + Standing rule 8: a number that does not tie to Odoo does not ship. These columns are minted + from the tenant MIRROR (`semantic.entity_measure_values`), so the oracle has to be the live + ERP and nothing derived from the mirror — otherwise both sides come from the same place and + the check cannot fail, which is the self-sealing shape `validate()`'s own header records + costing a wave. + + ⛔⛔ THE MIRROR IS BEHIND LIVE, ALWAYS, AND THAT IS NOT A DEFECT — so a bare equality here + would be RED every day and would teach everyone to ignore it. The reconciliation is therefore + two-legged, and the second leg is the one that carries the meaning: + + leg 1 totals agree within the lag, and the lag is REPORTED as a number, not a tolerance; + leg 2 ⭐ EVERY line-level difference traces to a line ODOO WROTE AFTER THE MIRROR'S OWN + WATERMARK. This is what makes the check falsifiable: a join bug produces differences + on lines the mirror holds perfectly, and leg 2 goes red on the first one. + + ⚠ Do NOT "fix" leg 2 by filtering the live side on `write_date <= watermark` and comparing + totals — MEASURED 2026-08-19, that is a far worse instrument: confirming an order touches its + lines' `write_date` without changing a value, so the filter drops thousands of lines the + mirror holds correctly and the gap grows from $425 to $140,030. + """ + from harness import datastore as DS + from harness import semantic as sem + + t = t or P.today() + checks = [] + try: + if not DS.ready(): + return [{"check": "product lookback measures reconcile to Odoo", + "ours": "no mirror", "theirs": "-", "ok": False, + "detail": "the tenant store is not readable, so the measures are UNPROVEN, " + "and an unproven aggregate is exactly what standing rule 8 bars"}] + except Exception as e: # noqa: BLE001 + return [{"check": "product lookback measures reconcile to Odoo", + "ours": f"{type(e).__name__}", "theirs": "-", "ok": False, "detail": str(e)[:200]}] + + # ⛔ THE WINDOW END COMES FROM THE MIRROR, NOT FROM `today`, and this was measured the wrong + # way round first. A window ending today reaches past what the mirror has ever seen: orders + # placed since the last sync exist live and NOWHERE in the store, so the totals leg reported + # a 2.69% "lag" that was really "the last three days do not exist here yet". Anchoring on the + # mirror's own newest order makes the comparison one about EDITS to a shared period — which + # is the only difference that could indicate a join bug. + con = DS.ro_cursor() + try: + _newest = con.execute("SELECT max(date_order) FROM sale_order").fetchone() + finally: + con.close() + d_to = t - _dt.timedelta(days=2) + if _newest and _newest[0]: + _n = str(_newest[0])[:10] + try: + # one day INSIDE the mirror's newest order: the final day may be half-synced. + d_to = min(d_to, _dt.date.fromisoformat(_n) - _dt.timedelta(days=1)) + except ValueError: + pass + d_from = d_to - _dt.timedelta(days=days) + DF, DT = d_from.isoformat(), d_to.isoformat() + + offer = sem.entity_measures("odoo_products") + checks.append({ + "check": "the product measure OFFER is non-empty and every key it names resolves in the " + "semantic model (owner item 4 / R1)", + "ours": len(offer), "theirs": ">0", "ok": len(offer) > 0, + "detail": {"keys": [m["key"] for m in offer], + # ⭐ The REPORTING half of standing rule 1: a declared key that dropped out + # says why, rather than being quietly absent from a list nobody diffs. + "refused": sem.entity_measure_refusals("odoo_products")}, + }) + if not offer: + return checks + missing_family = [m["key"] for m in offer if m.get("empty") not in ("zero", "blank")] + checks.append({ + "check": "every offered measure declares an EMPTY-WINDOW family (C1: additive->0, " + "ratio->blank), because 72% of this catalogue has no group in a 90-day window", + "ours": len(missing_family), "theirs": 0, "ok": not missing_family, + "detail": {"undeclared": missing_family}, + }) + + store = sem.entity_measure_values("odoo_products", ["revenue", "units", "margin"], + date_from=DF, date_to=DT, exclude_services=False, + offer=offer) + o = O.get_odoo() + # ⛔ ASKED OF ODOO DIRECTLY, grouped by Odoo's OWN product id — deliberately NOT by the SKU + # code the mirror joins on, so the oracle cannot inherit our join key. + g = o.read_group('sale.order.line', O.sale_line_domain(DF, DT), + ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum', 'margin:sum'], + ['product_id'], lazy=False) + live_tot = {"revenue": round(sum(r['price_subtotal'] for r in g), 2), + "units": round(sum(r['product_uom_qty'] for r in g), 2), + "margin": round(sum(r['margin'] for r in g), 2)} + ours_tot = {k: round(sum(v.get(k, 0) for v in store.values()), 2) + for k in ("revenue", "units", "margin")} + + # leg 2 — the falsifiable one. Every discrepant LINE must post-date the mirror's watermark. + con = DS.ro_cursor() + try: + wm = con.execute("SELECT cursor_wd FROM _sync_state WHERE entity = 'sale_order_line'" + ).fetchone() + wm = wm[0] if wm else None + rows = con.execute( + "SELECT l.id, l.price_subtotal FROM sale_order_line l " + "JOIN sale_order o ON o.id = l.order_id " + "WHERE o.state IN ('sale','done') AND o.team_id IN (5,6) " + " AND l.product_id IS NOT NULL " + " AND l.order_partner_id NOT IN " + " (SELECT id FROM res_partner WHERE name LIKE 'GIFTWARE%') " + " AND CAST(o.date_order AS TIMESTAMP) >= ? AND CAST(o.date_order AS TIMESTAMP) <= ?", + [f"{DF} 00:00:00", f"{DT} 23:59:59"]).fetchall() + finally: + con.close() + mine = {r[0]: (r[1] or 0.0) for r in rows} + theirs = {r['id']: (r['price_subtotal'] or 0.0) for r in + o.search_read('sale.order.line', O.sale_line_domain(DF, DT), + ['id', 'price_subtotal', 'write_date'])} + wd = {r['id']: str(r['write_date']) for r in + o.search_read('sale.order.line', O.sale_line_domain(DF, DT), ['id', 'write_date'])} + discrepant = [i for i, v in theirs.items() + if i not in mine or abs(mine[i] - v) >= 0.005] + unexplained = [i for i in discrepant if not wm or wd.get(i, '') <= str(wm)] + checks.append({ + "check": "every per-SKU measure difference vs live Odoo traces to a line Odoo wrote " + "AFTER the mirror's watermark, because a join bug would differ on a mirrored line", + "ours": len(unexplained), "theirs": 0, "ok": not unexplained, + "detail": {"lines_compared": len(theirs), "discrepant": len(discrepant), + "explained_by_mirror_lag": len(discrepant) - len(unexplained), + "watermark": str(wm), "window": [DF, DT], + "unexplained_line_ids": unexplained[:10]}, + }) + # ⛔⛔ THE LEG THAT CATCHES A WRONG JOIN KEY, and neither leg above can. Both of those sum the + # same LINES whatever dim they were grouped by, so swapping `dim: product_code` for + # `dim: product` (contract C1's trap, an Odoo id where the grid carries a SKU code) leaves + # them both green while every cell on the screen goes blank. This one asks: does the answer + # arrive under a key a GRID ROW ACTUALLY HAS, and is the value right FOR THAT SKU? + code_of = _codes_of_odoo_products(o, [r['product_id'][0] for r in g if r.get('product_id')]) + by_code = {} + for r in g: + if not r.get('product_id'): + continue + c = code_of.get(r['product_id'][0], f"pid:{r['product_id'][0]}") + by_code[c] = by_code.get(c, 0.0) + r['price_subtotal'] + # ⚠ The pool is the CALLER'S when it has one (it has already paid for it), and read fresh + # otherwise — `validate_measures` is runnable on its own, and a leg that silently skips when + # called directly is a leg nobody runs ([[gate-can-report-green-on-nothing]]). + pool_codes = set(pool_codes) if pool_codes is not None else { + r["code"] for r in pool(team_id=team_id, t=t)} + keyed_to_a_row = [c for c in store if c in pool_codes] + top = sorted(store.items(), key=lambda kv: -(kv[1].get("revenue") or 0))[:10] + spot = [{"sku": c, + "ours": round(v.get("revenue") or 0.0, 2), + "odoo": round(by_code.get(c, 0.0), 2)} for c, v in top] + # A named SKU may legitimately differ by a line Odoo edited after the watermark; the + # assertion is that MOST of the top ten tie exactly and NONE is off by an order of magnitude, + # which is what a mis-keyed join looks like (0.00 against a five-figure number). + exact = sum(1 for s in spot if abs(s["ours"] - s["odoo"]) < 0.005) + checks.append({ + "check": "the measure answer is KEYED TO THE GRID'S OWN IDENTITY (C1's join-key trap): " + "every group key is a SKU code a pool row carries, and the top-10 SKUs' revenue " + "ties to Odoo for THAT SKU", + "ours": {"keys_matching_a_pool_row": len(keyed_to_a_row), "of": len(store), + "top10_exact": exact}, + "theirs": {"keys_matching_a_pool_row": len(store), "of": len(store), "top10_exact": 10}, + # ⚠ Not `== len(store)`: a code whose ONLY product record is archived legitimately has + # revenue and no grid row (R12 keeps archived out), which `validate()`'s own + # "outside the grid" leg already reconciles. A mis-keyed join lands at ~0, not at 99%. + "ok": len(store) > 0 and len(keyed_to_a_row) / len(store) > 0.95 and exact >= 8, + "detail": {"spot_checks": spot, + "keys_with_no_pool_row": sorted(set(store) - pool_codes)[:10]}, + }) + checks.append({ + "check": "product lookback totals vs a DIRECT Odoo read_group, where the residual is mirror " + "lag and is REPORTED as a figure, never absorbed into a tolerance", + "ours": ours_tot, "theirs": live_tot, + # The assertion is on leg 2; this leg is red only if the lag is implausibly large, which + # is the shape that means "the mirror stopped" rather than "the mirror is a day behind". + "ok": abs(ours_tot["revenue"] - live_tot["revenue"]) <= max( + 0.01, live_tot["revenue"] * 0.02), + "detail": {"revenue_lag": round(ours_tot["revenue"] - live_tot["revenue"], 2), + "revenue_lag_pct": round( + (ours_tot["revenue"] - live_tot["revenue"]) / live_tot["revenue"] * 100, 4) + if live_tot["revenue"] else None, + "skus_in_grid_answer": len(store)}, + }) return checks diff --git a/platform/modules/products.py b/platform/modules/products.py index 459f07d937c81172a10eaeff383e1c98258744ea..c5b00db90081318bf415bee1c1b28ee2be9586e7 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -393,6 +393,178 @@ def pricelist_by_code(): return out, report +def active_products(limit=50000): + """`[{id, default_code, product_tmpl_id}]` for every ACTIVE product — read ONCE and shared. + + ⛔ RAISES ON A SHORT PULL, like `catalogue()` and for the same reason: a truncated product read + renders as a plausible smaller set of priced SKUs with nothing reporting it. + ⭐ It exists so `tier_prices_by_code` and `packagings_by_code` can be called from ONE pool + build without each paying for its own copy of the same 5,873-row read — measured at ~4 s each + on this connection, on a path that is somebody's first page load. + """ + dom = [('active', '=', True)] + prods = O.search_read('product.product', dom, ['id', 'default_code', 'product_tmpl_id'], + limit=limit) + n = O.get_odoo().search_count('product.product', dom) + if len(prods) != n: + raise ValueError(f"products.active_products: the product pull is TRUNCATED. Read " + f"{len(prods)} rows against a search_count of {n}.") + return prods + + +def tier_prices_by_code(prods=None): + """`({code: [{pricelist, unit_price}]}, report)` — EVERY live price a SKU really has. + + ⭐⭐ W37-T14 (owner: multiple prices per SKU). `pricelist_by_code` above answers a DIFFERENT + question and both are needed: it fills three DECLARED columns (`price_fisch`, `price_royal_1`, + `price_royal_2`) and therefore cannot show a price on a list the contract does not name. This + one is the honest set — one entry per pricelist that actually prices this SKU today. + + ⛔ IT READS EVERY LIVE PRICELIST, not `PRICELIST_COLUMNS`. Hardcoding this tenant's three list + names into "how many prices does a SKU have" is exactly what the ticket forbids, and it is what + would make the answer wrong for tenant #1 on the day they are onboarded. + + ⛔ ROWS ARE TEMPLATE-SCOPED (`proto/P2-pricing-uom.md`): joining on `product_id` drops 99.9% of + price rows, because `applied_on` is `1_product` on 10,454 of 10,468 items and `1_product` means + the TEMPLATE. A variant rule (`0_product_variant`, 13 rows) is the more specific statement and + wins, matching `pricecomp._tier_for` and `pricelist_by_code`. + + ⛔ FILTER BY `pricelist_id`, NEVER BY `active`: the default `product.pricelist.item` count hides + 2,580 archived items, so an `active` filter reads as a smaller, plausible, wrong set. + + ⚠ THE BASE TIER, at qty 1, for the same reason `pricelist_by_code` gives: a catalogue cell has + no quantity in hand. Measured: only 10 of 10,468 items carry a quantity break at all, so this + is very nearly the whole story rather than a simplification. + ⚠ CARDINALITY 1..3 TODAY, mode 3 — and `Royal 2` carries 2,605 price rows against **0 customers + and 0 orders**, so a SKU reading "3 tiers" is catalogue-true and commercially misleading. The + entry keeps the list NAME so a reader can see which tier it is rather than a bare count. + """ + report = {"lists": [], "rules_total": 0, "rules_not_fixed": 0, "rules_out_of_date": 0, + "rules_global": 0, "skus_with_no_price": 0, "identity_breaks": 0} + try: + pls = {p['id']: str(p.get('name') or '').strip() + for p in O.search_read('product.pricelist', [], ['id', 'name'])} + report["lists"] = sorted(pls.values()) + today = P.today().isoformat() + dom = [('pricelist_id', 'in', sorted(pls)), ('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product']), + '|', ('date_start', '=', False), ('date_start', '<=', today), + '|', ('date_end', '=', False), ('date_end', '>=', today), + ('fixed_price', '>', 0)] + rules = O.search_read('product.pricelist.item', dom, + ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', + 'fixed_price', 'min_quantity']) + + # R6's second sentence: what this reader cannot see is COUNTED, never dropped. + def _n(extra): + try: + return O.get_odoo().search_count('product.pricelist.item', extra) + except Exception: # noqa: BLE001 + return -1 # -1 reads as "not measured", never as zero + report["rules_total"] = _n([]) + report["rules_not_fixed"] = _n([('compute_price', '!=', 'fixed')]) + report["rules_global"] = _n([('applied_on', '=', '3_global')]) + report["rules_out_of_date"] = _n(['|', ('date_end', '!=', False), + ('date_start', '!=', False)]) + + by_var, by_tmpl = {}, {} + for r in rules: + plid = O.m2o_id(r.get('pricelist_id')) + if r.get('applied_on') == '0_product_variant' and r.get('product_id'): + by_var.setdefault((plid, O.m2o_id(r['product_id'])), []).append(r) + elif r.get('product_tmpl_id'): + by_tmpl.setdefault((plid, O.m2o_id(r['product_tmpl_id'])), []).append(r) + + prods = active_products() if prods is None else prods + except Exception as e: # noqa: BLE001 + report["error"] = f"{type(e).__name__}: {str(e)[:200]}" + return {}, report + + out = {} + for p in prods: + code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + tmpl = O.m2o_id(p.get('product_tmpl_id')) + tiers = [] + for plid, name in sorted(pls.items(), key=lambda kv: kv[1].lower()): + cands = by_var.get((plid, p['id'])) or by_tmpl.get((plid, tmpl)) + if not cands: + continue + base = min(cands, key=lambda r: r.get('min_quantity') or 0.0) + tiers.append({"pricelist": name, "unit_price": round(base.get('fixed_price') or 0.0, 2)}) + if tiers: + # ⚠ A code carried by TWO active products (D-309: `2112-12`) MERGES here, because the + # grid is keyed by code and one code is one row. The prices are UNIONED rather than + # one silently winning — a SKU that really does have two different Fisch prices should + # show both, and hiding one is how a $60 price disappeared behind a $24 one. + prior = out.get(code) + if prior is None: + out[code] = tiers + else: + seen = {(t["pricelist"], t["unit_price"]) for t in prior} + for t in tiers: + if (t["pricelist"], t["unit_price"]) not in seen: + prior.append(t) + report["identity_breaks"] += 1 + else: + report["skus_with_no_price"] += 1 + return out, report + + +def packagings_by_code(prods=None): + """`({code: [{name, qty}]}, report)` — the UNITS a SKU is really sold in (W37-T15). + + ⛔ IT COMES FROM `product.packaging` ALONE, and `proto/P2-pricing-uom.md` measured why the + obvious alternative is a dead end: the `uom.uom` Config category holds 26 conversion-bearing + units (`Case-Packed 12` … `Pallet-Packed 4,800`) referenced by **0 products, 0 sale lines and + 0 stock moves**, and every unit actually in use has `factor_inv = 1`. Building on `uom.uom` + conversions yields a column of 1s. + + ⛔ `qty > 1` IS A FILTER, NOT A TIDY-UP. 6,283 of 7,471 packaging rows are `qty = 1.0` — a + packaging that packs one of something is not a unit tier — plus real junk (one-character + names). Without it "units per SKU" reads **5,859** SKUs instead of **1,150**. + + ⛔ 133 ROWS ARE ORPHANS (`product_id = False`) and would collapse under a `groupby(product_id)` + into ONE fake product carrying 133 packagings. Dropped, and counted. + + ⚠ HONEST EMPTY: only **1,150 of 5,873 SKUs (19.6%)** have any unit tier, so this cell is + legitimately blank for 80% of the catalogue — "this SKU is sold in one unit", never "missing". + The 19.6% is not decoration: units are transacted on 77.9% of confirmed sale lines. + ⚠ `qty` is denominated in the product's OWN `uom_id` (matched on 7,338/7,338). + """ + report = {"rows_total": 0, "rows_orphan": 0, "rows_qty_le_1": 0, "skus_with_units": 0} + try: + rows = O.search_read('product.packaging', [], ['id', 'name', 'qty', 'product_id']) + except Exception as e: # noqa: BLE001 + report["error"] = f"{type(e).__name__}: {str(e)[:200]}" + return {}, report + report["rows_total"] = len(rows) + by_pid = {} + for r in rows: + pid = O.m2o_id(r.get('product_id')) + if not pid: + report["rows_orphan"] += 1 + continue + if (r.get('qty') or 0) <= 1: + report["rows_qty_le_1"] += 1 + continue + by_pid.setdefault(pid, []).append( + {"name": str(r.get('name') or '').strip(), "qty": float(r.get('qty') or 0)}) + try: + prods = active_products() if prods is None else prods + except Exception as e: # noqa: BLE001 + report["error"] = f"{type(e).__name__}: {str(e)[:200]}" + return {}, report + out = {} + for p in prods: + got = by_pid.get(p['id']) + if not got: + continue + code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + out.setdefault(code, []).extend(sorted(got, key=lambda u: u["qty"])) + report["skus_with_units"] = len(out) + return out, report + + def catalogue_count(): """The INDEPENDENT population oracle: Odoo's own count of active products. diff --git a/requirements.txt b/requirements.txt index 6b53d4a258e81dcdbe73f012a8ae135b136822e4..7468fc81ef71a091c8645b5d6dfcd49eafd0aa9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,47 +1,55 @@ -# AIOS web API — the FastAPI backend + the platform data-layer deps it reuses. -# -# ⛔ STREAMLIT IS GONE (EXIT-6, 2026-08-04) — do not add it back. The line here read -# `streamlit==1.58.0` with the comment "included only because the shared modules/core may import -# it at load; it is never served". That stopped being true when the shared layer was cleaned, and -# it stayed in THIS file — the one the Dockerfile actually installs — while `api/requirements.txt` -# had already dropped it and a gate reported the omission "verified". ~250 MB of image for a -# package nothing imported. -# -# The lesson is the file, not the package: the PINNED intent and the SHIPPED manifest are two -# different documents, and checking only the one you wrote is how the other rots. -# `api/verify_no_streamlit.py` now checks BOTH, and proves the API imports with streamlit, -# plotly, altair, openpyxl, reportlab and jinja2 all blocked at sys.meta_path. -# No altair/plotly/reportlab either — those were Streamlit UI/export only. openpyxl RETURNED in -# wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the -# import lives inside the handler, and the gate now asserts this line exists here. -fastapi>=0.139 -uvicorn[standard]>=0.30 -python-dotenv>=1.0 -pandas>=2.0 -requests>=2.28 -huggingface_hub>=0.20 -duckdb>=1.0 -pyyaml>=6.0 -pillow>=10.0 -beautifulsoup4>=4.12 -lxml>=5.0 -cryptography>=42.0 -# ⭐ WAVE 20 (R1 / D-4) — THE POSTGRES DRIVER, AND IT IS LOAD-BEARING IN THIS FILE SPECIFICALLY. -# `core/store_pg.py` is the store backend from this wave on, and it is deliberately FAIL-CLOSED: -# `_pool()` raises rather than falling back to the HF file store, so a container that gets -# `STORE_BACKEND=pg` without this line does not degrade — it refuses every request that touches -# the store, which is every authenticated request. -# -# ⛔ AND THIS IS THE FILE THAT MATTERS: the Dockerfile does `COPY requirements.txt` from the -# Space root, i.e. THIS manifest, not `api/requirements.txt`. That header's own streamlit story -# is the same defect in the other direction — the pinned intent and the shipped manifest are two -# documents. psycopg is now in BOTH, and `ops/verify_portability.py` gates the pair. -psycopg[binary,pool]>=3.2 -# ⭐ WAVE 21 (item 11, C5) — .xlsx preview for Select-from-file; lazy-imported in -# routes_uploads.py only. verify_no_streamlit asserts this line in BOTH manifests. -openpyxl>=3.1 -# ⛔ AND ITS TRANSPORT, learned from a RUNTIME_ERROR on the first v6 boot: FastAPI demands -# python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate -# was green because the dev box happens to have it for unrelated reasons — the container -# installs exactly this file. The environment-parity twin of the streamlit lesson above. -python-multipart>=0.0.20 +# AIOS web API — the FastAPI backend + the platform data-layer deps it reuses. +# +# ⛔ STREAMLIT IS GONE (EXIT-6, 2026-08-04) — do not add it back. The line here read +# `streamlit==1.58.0` with the comment "included only because the shared modules/core may import +# it at load; it is never served". That stopped being true when the shared layer was cleaned, and +# it stayed in THIS file — the one the Dockerfile actually installs — while `api/requirements.txt` +# had already dropped it and a gate reported the omission "verified". ~250 MB of image for a +# package nothing imported. +# +# The lesson is the file, not the package: the PINNED intent and the SHIPPED manifest are two +# different documents, and checking only the one you wrote is how the other rots. +# `api/verify_no_streamlit.py` now checks BOTH, and proves the API imports with streamlit, +# plotly, altair, openpyxl, reportlab and jinja2 all blocked at sys.meta_path. +# No altair/plotly/reportlab either — those were Streamlit UI/export only. openpyxl RETURNED in +# wave 21 (C5) as a LAZY dep of routes_uploads.py — the boot proof above still holds because the +# import lives inside the handler, and the gate now asserts this line exists here. +fastapi>=0.139 +uvicorn[standard]>=0.30 +python-dotenv>=1.0 +pandas>=2.0 +requests>=2.28 +huggingface_hub>=0.20 +duckdb>=1.0 +pyyaml>=6.0 +pillow>=10.0 +beautifulsoup4>=4.12 +lxml>=5.0 +cryptography>=42.0 +# ⭐ WAVE 20 (R1 / D-4) — THE POSTGRES DRIVER, AND IT IS LOAD-BEARING IN THIS FILE SPECIFICALLY. +# `core/store_pg.py` is the store backend from this wave on, and it is deliberately FAIL-CLOSED: +# `_pool()` raises rather than falling back to the HF file store, so a container that gets +# `STORE_BACKEND=pg` without this line does not degrade — it refuses every request that touches +# the store, which is every authenticated request. +# +# ⛔ AND THIS IS THE FILE THAT MATTERS: the Dockerfile does `COPY requirements.txt` from the +# Space root, i.e. THIS manifest, not `api/requirements.txt`. That header's own streamlit story +# is the same defect in the other direction — the pinned intent and the shipped manifest are two +# documents. psycopg is now in BOTH, and `ops/verify_portability.py` gates the pair. +psycopg[binary,pool]>=3.2 +# ⭐ WAVE 21 (item 11, C5) — .xlsx preview for Select-from-file; lazy-imported in +# routes_uploads.py only. verify_no_streamlit asserts this line in BOTH manifests. +openpyxl>=3.1 +# ⛔ AND ITS TRANSPORT, learned from a RUNTIME_ERROR on the first v6 boot: FastAPI demands +# python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate +# was green because the dev box happens to have it for unrelated reasons — the container +# installs exactly this file. The environment-parity twin of the streamlit lesson above. +python-multipart>=0.0.20 +# ⭐ WAVE 37 (D-346, answering `ASK E-1`) — THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the +# Messages API over raw `requests`, which meant re-implementing retries, streaming and error +# taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the +# raw-HTTP call still there as the fallback, so the feature works with or without this line — what +# the line does is turn the fallback into dead weight instead of the live path. +# ⚠ BOTH MANIFESTS, NOT ONE. This file's own streamlit story at the top is exactly what pinning +# one of them looks like a year later. +anthropic>=0.96 diff --git a/web/index.html b/web/index.html index af3619ad7cada483ab9b0b64caf178ce69a9abdb..cf3ad791ca9b8c6918372a1c90927a7b2ef2236e 100644 --- a/web/index.html +++ b/web/index.html @@ -1,14 +1,46 @@ - - - - - - - Loopable - - -
-
- - - + + + + + + + Loopable + + + +
+
+ + + diff --git a/web/src/App.tsx b/web/src/App.tsx index 77fd5fb03fc59405df22b003bfed1a8bb3b539f7..8e740b30aa739af89fe5978458176d8cc964a6e6 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,30 +1,41 @@ -import CustomerGrid from "./customer-grid/CustomerGrid"; -import { OverlayProvider } from "./customer-grid/OverlaySurface"; -import { isStreamlitComponent } from "./customer-grid/hostBridge"; -import Shell from "./shell/Shell"; - -// HOSTED = either Streamlit path (the component iframe, or the legacy injected -// payload). There the surrounding app owns all chrome and this tree renders the -// BARE grid, byte-for-byte the behavior the embed always had. Only the -// standalone build gets the shell (wave 4, the strangler frame). Computed once -// at module level, like useCustomerData's own mode constants — the host mode -// cannot change within a page lifetime. -// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to -// install the standalone event sink, and two copies of a mode predicate is how -// the two halves of "standalone-only" end up disagreeing. -export const HOSTED = - isStreamlitComponent() || - typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; - -export default function App() { - if (HOSTED) { - return ( -
- - - -
- ); - } - return ; -} +import { Suspense, lazy } from "react"; +// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 +// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the +// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build +// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, +// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. +// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is +// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is +// indistinguishable from a broken one. +const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); +import { OverlayProvider } from "./customer-grid/OverlaySurface"; +import { isStreamlitComponent } from "./customer-grid/hostBridge"; +import Shell from "./shell/Shell"; + +// HOSTED = either Streamlit path (the component iframe, or the legacy injected +// payload). There the surrounding app owns all chrome and this tree renders the +// BARE grid, byte-for-byte the behavior the embed always had. Only the +// standalone build gets the shell (wave 4, the strangler frame). Computed once +// at module level, like useCustomerData's own mode constants — the host mode +// cannot change within a page lifetime. +// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to +// install the standalone event sink, and two copies of a mode predicate is how +// the two halves of "standalone-only" end up disagreeing. +export const HOSTED = + isStreamlitComponent() || + typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; + +export default function App() { + if (HOSTED) { + return ( +
+ + + +
+ ); + } + return ; +} diff --git a/web/src/account/SubscriptionPage.tsx b/web/src/account/SubscriptionPage.tsx index 3b815ab4b0fb47791358f302caa942923542e572..59f00a708e4ab698bd6e9863238898e970f0a56c 100644 --- a/web/src/account/SubscriptionPage.tsx +++ b/web/src/account/SubscriptionPage.tsx @@ -1,47 +1,49 @@ -// --------------------------------------------------------------------------- -// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). -// -// Owner item 13, verbatim: *"add below settings the following button: Feedback …, -// Usage credits …, and a Subscription module … Specialized for the Royal Imports -// tenant, do not show it."* -// -// This is the third of those three, and the only one with nothing behind it yet. -// So it says so, in the words this app already uses for a boarded door -// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later -// release."). One voice for one state — DESIGN.md 1's "variety is a defect". -// -// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. -// C6 puts the Royal Imports rule in TWO places that are both the frame's: the -// account MENU does not draw the row, and the ROUTE refuses. A third opinion here -// would be a third thing to keep in step, and the day they disagree the reader -// gets a menu row that opens a page which says it does not exist. The page renders -// the same for every tenant that can reach it, because reaching it IS the decision. -// -// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. -// The wave's rule is that a prop must be REQUIRED, never optional — an optional one -// degrades to "the page does not exist", which is indistinguishable from never -// built. That rule constrains the props that EXIST; it does not ask for a prop that -// carries nothing. There is nothing this page needs from the frame: it holds no -// state, makes no call, and takes no decision the frame has already taken. A -// `tenant` prop here would be exactly the second opinion the paragraph above -// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. -// --------------------------------------------------------------------------- - -import "./account.css"; - -export default function SubscriptionPage() { - return ( -
-

Subscription

-
- {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not - narrate itself). The reader arrived here on purpose and wants one fact: is there - anything to do? There is not, and every further word would be spent on someone - who is already leaving. No feature list, no roadmap, no "in the meantime". */} -

- Under construction. Plans, billing and invoices will open here in a later release. -

-
-
- ); -} +// --------------------------------------------------------------------------- +// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). +// +// Owner item 13, verbatim: *"add below settings the following button: Feedback …, +// Usage credits …, and a Subscription module … Specialized for the Royal Imports +// tenant, do not show it."* +// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the +// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. +// +// This is the third of those three, and the only one with nothing behind it yet. +// So it says so, in the words this app already uses for a boarded door +// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later +// release."). One voice for one state — DESIGN.md 1's "variety is a defect". +// +// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. +// C6 puts the Royal Imports rule in TWO places that are both the frame's: the +// account MENU does not draw the row, and the ROUTE refuses. A third opinion here +// would be a third thing to keep in step, and the day they disagree the reader +// gets a menu row that opens a page which says it does not exist. The page renders +// the same for every tenant that can reach it, because reaching it IS the decision. +// +// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. +// The wave's rule is that a prop must be REQUIRED, never optional — an optional one +// degrades to "the page does not exist", which is indistinguishable from never +// built. That rule constrains the props that EXIST; it does not ask for a prop that +// carries nothing. There is nothing this page needs from the frame: it holds no +// state, makes no call, and takes no decision the frame has already taken. A +// `tenant` prop here would be exactly the second opinion the paragraph above +// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. +// --------------------------------------------------------------------------- + +import "./account.css"; + +export default function SubscriptionPage() { + return ( +
+

Subscription

+
+ {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not + narrate itself). The reader arrived here on purpose and wants one fact: is there + anything to do? There is not, and every further word would be spent on someone + who is already leaving. No feature list, no roadmap, no "in the meantime". */} +

+ Under construction. Plans, billing and invoices will open here in a later release. +

+
+
+ ); +} diff --git a/web/src/account/UsagePage.tsx b/web/src/account/UsagePage.tsx index f2d6dedc39b6b29f11d7d0b43aa8ab81a710acb6..6ab2ceb32d8c3e1213c3df4e9667466d4384d916 100644 --- a/web/src/account/UsagePage.tsx +++ b/web/src/account/UsagePage.tsx @@ -1,297 +1,297 @@ -// --------------------------------------------------------------------------- -// account/UsagePage.tsx — WAVE 35 · W35-T19 (owner item 13, ruling R9, C6/C7). -// -// R9: ONE meter for every AI surface. It REPORTS weekly usage against an -// allowance and does not cut anyone off this wave; over the allowance the -// product still works and the bar is red. -// -// ⛔⛔ THIS PAGE MAY NOT INVENT A FIGURE, AND THAT IS THE WHOLE TICKET. -// Every number here comes off `GET /usage` — the week, the reset date, the -// allowance, each surface's tokens and calls. Nothing is derived from a clock, -// nothing is estimated, and a surface with no ledger lines shows **0**, not a -// blank and not an omission (E-9 guarantees all four surfaces are always -// present, so there is no missing-row branch to get wrong). -// [[no-unverifiable-aggregates]] -// -// ⛔ AND `unmeasured` IS RENDERED, NOT SWALLOWED. Those are real calls whose -// provider declined to report a token count, so the total beside them is a -// FLOOR. A page that hides them presents the floor as a complete figure, which -// is the cost-surprise failure one step removed — the same reason -// `usage_ledger` books an unmeasured call as unknown rather than as zero. -// -// ⚠ SURFACE/READ SPLIT, as on Home and Starred: `renderToStaticMarkup` runs no -// effect, so a shot of the fetching component photographs only its spinner. -// --------------------------------------------------------------------------- - -import { useCallback, useEffect, useRef, useState } from "react"; - -import { fmt } from "../ui/fmt"; -import { loadUsage } from "./accountApi"; -import type { Usage, UsageSurface } from "./accountApi"; -import "./account.css"; - -/** - * The four surfaces C7 names, in the order R9 lists them. - * - * ⚠ AN UNKNOWN KEY IS HUMANISED, NEVER DROPPED. Dropping it would hide real usage from a meter - * whose whole job is to account for it, and printing the raw key would put an internal identifier - * on screen. The day a fifth LLM entry point lands (this product added two in one wave) it shows - * up here as a readable name with the right number beside it, and somebody can then decide what - * to call it. - */ -const SURFACE_LABEL: Record = { - assistant: "Assistant", - field_agent: "Field agents", - ai_review: "Reviews", - automation_draft: "Drafting", -}; -const ORDER = ["assistant", "field_agent", "ai_review", "automation_draft"]; - -export function surfaceLabel(key: string): string { - const known = SURFACE_LABEL[key]; - if (known) return known; - const words = key.replace(/[_-]+/g, " ").trim(); - return words ? words.charAt(0).toUpperCase() + words.slice(1) : key; -} - -/** C7's order first, then anything the server added, alphabetically. */ -export function orderSurfaces(surfaces: UsageSurface[]): UsageSurface[] { - const rank = (s: UsageSurface) => { - const i = ORDER.indexOf(s.surface); - return i === -1 ? ORDER.length : i; - }; - return [...surfaces].sort( - (a, b) => rank(a) - rank(b) || surfaceLabel(a.surface).localeCompare(surfaceLabel(b.surface)) - ); -} - -/** - * The bar's fill, as a percentage, CLAMPED to 100. - * - * ⚠ The clamp is what keeps "over the allowance" a COLOUR rather than a bar that overflows its - * own track and paints across the page. The number above the bar is unclamped and is where the - * overage is actually read. - * ⚠ An allowance of 0 (unset) yields 0 rather than a division by zero: no allowance means the - * meter has nothing to measure against, and a full red bar would be an assertion nobody made. - */ -export function barPct(value: number, of: number): number { - if (!(of > 0)) return 0; - return Math.max(0, Math.min(100, (value / of) * 100)); -} - -export type UsageLoad = - | { phase: "loading" } - | { phase: "ready"; usage: Usage } - | { phase: "failed"; message: string }; - -function Bar({ pct, over }: { pct: number; over: boolean }) { - return ( -